def test_batch_norm_inference():
num_examples = 1000
data = saved_data[20]
assert len(data) == 15
x = data[0]
y = data[1]
soldbeta = data[2]
soldgamma = data[3]
xs = data[4]
solground = data[5:]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()],
weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=1)
batch_size = 100
mlp.train()
for b in range(0, 1):
mlp.zero_grads()
mlp.forward(x[b:b + batch_size])
mlp.backward(y[b:b + batch_size])
mlp.step()
closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta")
closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
for b in range(0, num_examples, batch_size):
mlp.eval()
student = mlp.forward(xs[b:b + batch_size])
ground = solground[b//batch_size]
closeness_test(student, ground, "mlp.forward(x)")
def predict(self, input_data):
A1 = np.dot(self.input_data, self.W1) + self.B1
Z1 = activation.Sigmoid(A1)
A2 = np.dot(Z1, self.W2) + self.B2
y = activation.Sigmoid(A2)
predicted_num = np.argmax(y)
return predicted_num
def test_batch_norm_train():
data = saved_data[19]
assert len(data) == 10
x = data[0]
y = data[1]
soldW = data[2:5]
soldb = data[5:8]
soldbeta = data[8]
soldgamma = data[9]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()],
weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=1)
mlp.forward(x)
mlp.backward(y)
dW = [x.dW for x in mlp.linear_layers]
db = [x.db for x in mlp.linear_layers]
for i, (pred, gt) in enumerate(zip(dW, soldW)):
closeness_test(pred, gt, "mlp.dW[%d]" % i)
for i, (pred, gt) in enumerate(zip(db, soldb)):
closeness_test(pred, gt, "mlp.db[%d]" % i)
closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta")
closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
def test_momentum():
data = saved_data[21]
assert len(data) == 8
x = data[0]
y = data[1]
solW = data[2:5]
solb = data[5:]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.856, num_bn_layers=0)
num_test_updates = 5
for u in range(num_test_updates):
mlp.zero_grads()
mlp.forward(x)
mlp.backward(y)
mlp.step()
mlp.eval()
W = [x.W for x in mlp.linear_layers]
b = [x.b for x in mlp.linear_layers]
for i, (pred, gt) in enumerate(zip(W, solW)):
closeness_test(pred, gt, "mlp.linear_layers[%d].W" % i)
for i, (pred, gt) in enumerate(zip(b, solb)):
closeness_test(pred, gt, "mlp.linear_layers[%d].b" % i)
def test_mystery_hidden_backward2():
data = saved_data[17]
assert len(data) == 14
x = data[0]
y = data[1]
soldW = data[2:8]
soldb = data[8:]
reset_prng()
mlp = hw1.MLP(784,
10, [32, 32, 32, 32, 32], [
activation.Sigmoid(),
activation.Sigmoid(),
activation.Sigmoid(),
activation.Sigmoid(),
activation.Sigmoid(),
activation.Identity()
],
weight_init,
bias_init,
loss.SoftmaxCrossEntropy(),
0.008,
momentum=0.0,
num_bn_layers=0)
mlp.forward(x)
mlp.backward(y)
dW = [x.dW for x in mlp.linear_layers]
db = [x.db for x in mlp.linear_layers]
for i, (pred, gt) in enumerate(zip(dW, soldW)):
closeness_test(pred, gt, "mlp.linear_layers[%d].dW" % i)
for i, (pred, gt) in enumerate(zip(db, soldb)):
closeness_test(pred, gt, "mlp.linear_layers[%d].db" % i)
def test_mystery_hidden_forward1():
data = saved_data[13]
x = data[0]
gt = data[1]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()],
weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=0)
pred = mlp.forward(x)
closeness_test(pred, gt, "mlp.forward(x)")
def feed_forward(self, W1, W2, B1, B2):
delta = 1e-7 #log 무한대 발산 방지를 위해 추가한다.
A1 = np.dot(self.input_data, W1) + B1
Z1 = activation.Sigmoid(A1)
A2 = np.dot(Z1, W2) + B2
y = activation.Sigmoid(A2)
# 로그 최대 우도 추정법을 사용한다.
return -np.sum(self.target_data * np.log(y + delta) +
(1 - self.target_data) * np.log((1 - y) + delta))
def cost(self):
delta = 1e-7 # log 무한대 발산 방지를 위해 추가한다.
A1 = np.dot(self.input_data, self.W1) + self.B1
Z1 = activation.Sigmoid(A1)
A2 = np.dot(Z1, self.W2) + self.B2
y = activation.Sigmoid(A2)
# 로그 최대 우도 추정법을 사용한다.
cost_val = -np.sum(self.target_data * np.log(y + delta) +
(1 - self.target_data) * np.log((1 - y) + delta))
return cost_val
def test_sigmoid_forward():
data = saved_data[5]
t0 = data[0]
gt = data[1]
student = activation.Sigmoid()
student(t0)
closeness_test(student.state, gt, "sigmoid.state")
def test_sigmoid_derivative():
data = saved_data[6]
t0 = data[0]
gt = data[1]
student = activation.Sigmoid()
student(t0)
closeness_test(student.derivative(), gt, "sigmoid.derivative()")
def test_sigmoid():
print("gradient check: Sigmoid")
x = np.random.rand(5 * 8).reshape((5, 8)).astype('float32')
y = np.random.rand(5 * 8).reshape((5, 8)).astype('float32')
sigmoid = activation.Sigmoid()
sqaure_loss_func = loss.SquareLoss()
sigmoid_x = sigmoid(x)
square_loss = sqaure_loss_func(sigmoid_x, y)
torch_x = torch.Tensor(x)
torch_x.requires_grad = True
sigmoid_torch = nn.Sigmoid()
square_loss_func_torch = nn.MSELoss()
sigmoid_x_torch = sigmoid_torch(torch_x)
sqaure_loss_torch = square_loss_func_torch(sigmoid_x_torch,
torch.Tensor(y))
print("Value:\ntorch:{},mine:{}, delta:{}".format(
sqaure_loss_torch.item(), square_loss,
(sqaure_loss_torch.item() - square_loss)))
# --- my grad ---
grad_sigmoid = sqaure_loss_func.backward()
grad_x = sigmoid.backward(grad_sigmoid)
# --- torch grad ---
sqaure_loss_torch.backward()
grad_x_torch = torch_x.grad.data.numpy()
print(grad_x_torch - grad_x)
def test_sigmoid():
s = activation.Sigmoid()
print(s(0))
print(s(-3))
print(s(3))
print(s(500))
print(s(-1000))
print(
s(-999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
))
def __init__(self):
self.weights = None
self.weightdeltas = None
self.inputsum = None
# Default hidden node activation is sigmoidal. To change,
# simply set neuron.activation to any of the activation
# functions in the activation module after the neuron has been
# instantiated.
self.activation = activation.Sigmoid()
def main():
np.random.seed(1)
alpha = 0.03 # 学习率 learning rate
sigmoid = activation.Sigmoid()
a0, y = loadData() # A,B,C,D
# 1,1,1,0
# 0,1,1,1
# 0,0,1,0
# 1,0,1,1
# 0,0,0,0
# 0,1,0,1
# 1,1,0,0
# 1,0,0,1
# 每层节点个数：l0-3, l1-4, l2-1，可以确定权重矩阵的维度
w1 = generateWeights((3, 4))
w2 = generateWeights((4, 1))
for i in range(100000):
z1 = np.dot(a0, w1) # 正向过程：每一级的输出都是下一级的输入
a1 = sigmoid(z1) #
z2 = np.dot(a1, w2)
a2 = sigmoid(z2)
loss = Loss(a2, y) # 反向过程，由后至前反推一遍
grad_a2 = loss.gradient()
grad_z2 = grad_a2 * sigmoid.gradient(z2)
grad_w2 = np.dot(a1.T, grad_z2) / len(a1) # 成本函数对应的是平均值
grad_a1 = np.dot(grad_z2, w2.T) # 死公式，但很有意思
# 这一步非常重要，而且不太好理解，最好能多通过流程图在稿纸上多演算几次运算过程和矩阵乘法
grad_z1 = grad_a1 * sigmoid.gradient(z1)
grad_w1 = np.dot(a0.T, grad_z1) / len(a0)
# 看反向过程的每一层都有求da, dz, dw。而且基本都是死公式
w1 -= alpha * grad_w1
w2 -= alpha * grad_w2
# 验证
s0 = [[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0], [1, 0, 1],
[1, 1, 0], [1, 1, 1]]
s1 = np.dot(s0, w1)
s2 = sigmoid(s1)
s3 = np.dot(s2, w2)
s4 = sigmoid(s3)
print(s4)
def main():
np.random.seed(1)
a0, y = loadData()
w0, b0 = layer.Layer.randomW_B(3, 4)
w1, b1 = layer.Layer.randomW_B(4, 1)
l0 = layer.Layer(
w0,
b0,
learning_rate=0.3,
act=activation.LeakyReLU()
)
l1 = layer.Layer(
w1,
b1,
learning_rate=0.3,
act=activation.Sigmoid()
)
for i in range(10000):
a1 = l0.fp(a0)
a2 = l1.fp(a1)
loss = Loss(a2, y) # 反向过程，由后至前反推一遍
g_a2 = loss.gradient()
g_a1 = l1.bp(g_a2)
l0.bp(g_a1)
# 验证
s0 = [
[0, 0, 0],
[0, 0, 1],
[0, 1, 0],
[0, 1, 1],
[1, 0, 0],
[1, 0, 1],
[1, 1, 0],
[1, 1, 1]]
s1 = l0.fp(s0)
s2 = l1.fp(s1)
print(s2)
def __init__(self,
is_leaky_relu: bool = False,
leakage_coeff: float = 0.2):
self._relu = activation.ReLU(
is_leaky=is_leaky_relu,
leakage=leakage_coeff if is_leaky_relu else 0)
self._sigmoid = activation.Sigmoid()
self._identity = activation.Identity()
self._network = [[
Perceptron(num_inputs, self._relu)
for _ in range(hiddenlayer1_size)
],
[
Perceptron(hiddenlayer1_size, self._relu)
for _ in range(hiddenlayer2_size)
],
[
Perceptron(hiddenlayer2_size, self._sigmoid)
for _ in range(num_outputs)
]]
self._mn_data = MNIST('./images')
self._desired_changes = None
self._layer_inputs = None
def learnNN(chromaNorm = 'L1', constantQNorm = 'Linf', deltaTrain = 2, nnStruct = [256, 50, 24], errorFunc = 'SSE', verbose = False):
'''
Learns neural network weights with unbuffered feature input: store all features in main memory and do one giant batch train.
PARAMETERS
----------
chromaNorm: L1, L2, Linf, or None, normalization of chroma
constantQNorm: L1, L2, Linf, or None, normalization of constant q transform
nnStruct {List}: neural network layer list (each element is number of neurons at the layer
errorFunc {String}: 'KLDiv' or 'SSE'
verbose {Boolean}: report progress to standard out
RETURN
------
net: trained neural network
'''
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct)-2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
if verbose:
print "Retrieving Features."
# read constant-q transform preliminary features
Xtrain = np.loadtxt('data/logfreqspec.csv', dtype=np.float, delimiter=',', usecols=range(1,257))
# read bass and treble chromagram features
Xtarget = np.loadtxt('data/bothchroma.csv', dtype=np.float, delimiter=',', usecols=range(1,25))
if verbose:
print "Normalizing Features."
divInd = np.sum(Xtrain, axis=1) != 0
# perform feature normalization
if constantQNorm == 'L1':
Xtrain[divInd,:] /= np.sum(np.abs(Xtrain[divInd,:]), axis=1)[:,np.newaxis]
elif constantQNorm == 'L2':
Xtrain[divInd,:] /= np.sum(Xtrain[divInd,:] ** 2, axis=1)[:,np.newaxis]
elif constantQNorm == 'Linf':
Xtrain[divInd,:] /= np.max(np.abs(Xtrain[divInd,:]), axis=1)[:,np.newaxis]
del divInd
divIndTreble = np.sum(Xtarget[:,0:12], axis=1) != 0
divIndBass = np.sum(Xtarget[:,12:24], axis=1) != 0
# perform feature normalization
if chromaNorm == 'L1':
Xtarget[divIndTreble,0:12] /= np.sum(np.abs(Xtarget[divIndTreble,0:12]), axis=1)[:,np.newaxis]
Xtarget[divIndBass,12:24] /= np.sum(np.abs(Xtarget[divIndBass,12:24]), axis=1)[:,np.newaxis]
elif chromaNorm == 'L2':
Xtarget[divIndTreble,0:12] /= np.sum(Xtarget[divIndTreble,0:12] ** 2, axis=1)[:,np.newaxis]
Xtarget[divIndBass,12:24] /= np.sum(Xtarget[divIndBass,12:24] ** 2, axis=1)[:,np.newaxis]
elif chromaNorm == 'Linf':
Xtarget[divIndTreble,0:12] /= np.max(np.abs(Xtarget[divIndTreble,0:12]), axis=1)[:,np.newaxis]
Xtarget[divIndBass,12:24] /= np.max(np.abs(Xtarget[divIndBass,12:24]), axis=1)[:,np.newaxis]
del divIndTreble
del divIndBass
# batch train Neural Network
trainNet(Xtrain, Xtarget, net, errorFunc, verbose)
if verbose:
print "All done!"
return net
import activation as act import cost_functions as cost from data_prep import DataPrep from NeuralNetwork import DenseLayer, NeuralNetwork from NN_keras import Keras # set the parameters n = 100 n_epochs = 300 n_batches = 100 eta = 0.5 lmbda = 0 neurons = [10, 10] n_outputs = 1 hidden_act = act.Sigmoid() output_act = act.Identity() # create data using franke function seed = 2034 np.random.seed(seed) x = np.sort(np.random.uniform(0, 1, n)) y = np.sort(np.random.uniform(0, 1, n)) x, y = np.meshgrid(x, y) z = np.ravel(f.FrankeFunction(x, y) + 0.1*np.random.randn(x.shape[0], x.shape[1])) z = z.reshape(-1, 1) # set up the design matrix data = DataPrep() X = data.design_matrix(x, y, degree=1)[:, 1:]
def f(x):
return (0.5 * np.sin(x)) + 0.5, (0.3 * np.cos(x)) + 0.3
# create train samples
xtrain = np.linspace(-7, 7, numTrain).reshape(numTrain, 1)
ytrain = np.zeros([numTrain, 2])
ytrain[:, 0] = f(xtrain)[0].squeeze()
ytrain[:, 1] = f(xtrain)[1].squeeze()
# Instantiate neural network with 2 layers.
# The hidden layer has 5 neurons, with 1 output.
# assumes full connectivity between layer neurons.
# uses sigmoid activation function by default at each layer.
net = nn.NeuralNet([1, 100, 50, 2],
actFunc=[act.Sigmoid(),
act.Sigmoid(),
act.ArcTan()])
# hire a trainer for the network
trainer = nn.Trainer(net, 'SSE', 25, xtrain, ytrain)
# train using BFGS and sum of squares error
#optArgs = {'gtol': 1e-6, 'maxiter': 250}
#trainer.trainBFGS(**optArgs)
optArgs = {
'bounds': None,
'm': 1000,
'factr': 1e7,
'pgtol': 1e-02,
def __register_unary_math_op__(op_name, act):
def op(input, name=None):
return layer.mixed(input=[layer.identity_projection(input=input)],
name=name,
act=act)
op = wrap_name_default(op_name)(op)
op.__doc__ = type(act).__doc__
globals()[op_name] = op
__all__.append(op_name)
__register_unary_math_op__('exp', act.Exp())
__register_unary_math_op__('log', act.Log())
__register_unary_math_op__('abs', act.Abs())
__register_unary_math_op__('sigmoid', act.Sigmoid())
__register_unary_math_op__('tanh', act.Tanh())
__register_unary_math_op__('square', act.Square())
__register_unary_math_op__('relu', act.Relu())
__register_unary_math_op__('sqrt', act.Sqrt())
__register_unary_math_op__('reciprocal', act.Reciprocal())
__register_unary_math_op__('softmax', act.Softmax())
def __add__(layeroutput, other):
if is_compatible_with(other, float):
return layer.slope_intercept(input=layeroutput, intercept=other)
if not isinstance(other, Layer):
raise TypeError("Layer can only be added with"
" another Layer or a number")
if layeroutput.size == other.size:
if choice == 0:
epochs = int(input("Please input how many epochs you want to train over [DEFAULT = 6000] : ") or "6000")
print("")
criterion = loss.LossBCE()
model = sequential.Sequential(
layer.Linear(2, 25),
activation.ReLU(),
layer.Linear(25, 50),
activation.ReLU(),
layer.Linear(50, 50),
activation.ReLU(),
layer.Linear(50, 25),
activation.ReLU(),
layer.Linear(25, 1),
activation.Sigmoid()
)
train_target[((train_input-0.5)**2).sum(1) < 1/(2*math.pi)] = 0
train_target[((train_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1
test_target[((test_input-0.5)**2).sum(1) < 1/(2*math.pi)] = 0
test_target[((test_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1
ps = model.parameters()
optim = optimizer.SGD(model.parameters(), lr=0.05, decay=500)
#for plotting later
levels = [0, 0.25, 0.5, 0.75, 1]
#for accuracy computation
sigmoid = True
else:
epochs = int(input("Please input how many epochs you want to train over [DEFAULT = 4000] : ") or "4000")
print("")
def test_activations():
check_activation(activation.Sigmoid())
check_activation(activation.ReLU())
def testNNFeature(getSong=1, chromaNorm='L1', constantQNorm='L1'):
# initialize feature storage
Xtrain = [] # Constant-Q transform
Xtarget = [] # Bass and treble chromagram
nnStruct = [256, 24]
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct) - 2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
# load in trained weights
wstar = np.load("trainedweights/wstar_grad_KLDiv_[0]_0.75_100iter.npy")
net.setWeights(wstar)
# read constant-q transform preliminary features
qFile = open('data/logfreqspec.csv', 'r')
# read bass and treble chromagram features
cFile = open('data/bothchroma.csv', 'r')
songNum = 0
songPath = ''
for cObs, qObs in izip(cFile, qFile):
cObs = cObs.split(",")
qObs = qObs.split(",")
# check if we have moved to a new song
if cObs[0]:
# check features are in sync by audio file path
if not qObs[0] or cObs[0] != qObs[0]:
raise ValueError("Feature files out of sync")
# run gathered features through the neural net and return the values
if songNum > 0 and songNum == getSong:
print "Processing song #%d, %s" % (songNum, songPath)
train = np.asarray(Xtrain)
target = np.asarray(Xtarget)
output = net.calcOutput(train)
return train, output, target
songNum += 1
songPath = cObs[0]
if songNum != getSong:
continue
# double check features are in sync by timestamp
if float(cObs[1]) != float(qObs[1]):
raise ValueError("Feature files out of sync")
# get chromagrams
chroma = np.asfarray(cObs[2:])
# avoid divide by zero (this is silence in audio)
#if np.sum(chroma) == 0:
# continue
# perform feature normalization
if chromaNorm == 'L1':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(np.abs(chroma[12:24]))
elif chromaNorm == 'L2':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(chroma[0:12]**2)
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(chroma[12:24]**2)
elif chromaNorm == 'Linf':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.max(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.max(np.abs(chroma[12:24]))
Xtarget.append(chroma)
# get Constant-Q transform
constantQ = np.asfarray(qObs[2:])
# perform feature normalization
if constantQNorm is not None and np.sum(constantQ) != 0:
if constantQNorm == 'L1':
constantQ /= np.sum(np.abs(constantQ))
elif constantQNorm == 'L2':
constantQ /= np.sum(constantQ**2)
elif constantQNorm == 'Linf':
constantQ /= np.max(np.abs(constantQ))
Xtrain.append(constantQ)
def mixlearnNNbuff(chromaNorm='L1',
constantQNorm=None,
deltaTrain=2,
nnStruct=[256, 150, 24],
errorFunc='SSE',
verbose=False,
numDataPass=1):
'''
Learns neural network weights with buffered feature input (batch training in segments).
Use this function when thrashing to disk is a possibility
PARAMETERS
----------
chromaNorm: L1, L2, Linf, or None, normalization of chroma
constantQNorm: L1, L2, Linf, or None, normalization of constant q transform
deltaTrain {int}: how many songs to train after. Buffering features prevents thrashing to disk.
nnStruct {List}: neural network layer list (each element is number of neurons at the layer
errorFunc {String}: 'KLDiv' or 'SSE'
verbose {Boolean}: report progress to standard out
RETURN
------
net: trained neural network
'''
# initialize feature storage
Xtrain = [] # Constant-Q transform
Xtarget = [] # Bass and treble chromagram
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct) - 2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
# hire a trainer for the network
trainer = nn.Trainer(net, errorFunc, 1)
for iDataset in range(numDataPass):
print "DATASET PASS %d" % (iDataset + 1)
# get feature file pointers
qtransFiles, chromaFiles = process_dir("data/burgoyne2011chords")
# until all the feature files have been exhausted
passInd = 0
while len(qtransFiles) > 0 and len(chromaFiles) > 0:
# for each pair of feature files that remain
i = 0
for qFile, cFile in izip(qtransFiles[:], chromaFiles[:]):
# open Constant-Q file and restore reading offset
qFilePtx = open(qFile["path"], 'r')
qFilePtx.seek(qFile["offset"])
# read an observation
qObs = qFilePtx.readline().strip()
# we've exhausted the observations in this song
if not qObs:
print "DONE WITH SONG: %s" % qFile["path"]
# remove from the processing queue
del qtransFiles[i]
del chromaFiles[i]
continue
# update offset
qtransFiles[i]["offset"] = qFilePtx.tell()
# close the file pointer
qFilePtx.close()
# open chroma file and restore reading offset
cFilePtx = open(cFile["path"], 'r')
cFilePtx.seek(cFile["offset"])
# read an observation
cObs = cFilePtx.readline().strip()
# update offset
chromaFiles[i]["offset"] = cFilePtx.tell()
# close the file pointer
cFilePtx.close()
i += 1
if passInd < 50:
continue
qObs = qObs.split(",")
cObs = cObs.split(",")
# check features are in sync by timestamp
if float(cObs[0]) != float(qObs[0]):
raise ValueError("Feature files out of sync")
# get chromagrams
chroma = np.asfarray(cObs[1:])
# avoid divide by zero (this is silence in audio)
if np.sum(chroma) < 3.0:
continue
# perform feature normalization
if chromaNorm == 'L1':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(np.abs(chroma[12:24]))
elif chromaNorm == 'L2':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(chroma[0:12]**2)
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(chroma[12:24]**2)
elif chromaNorm == 'Linf':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.max(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.max(np.abs(chroma[12:24]))
Xtarget.append(chroma)
# get Constant-Q transform
constantQ = np.asfarray(qObs[1:])
# perform feature normalization
if constantQNorm is not None and np.sum(constantQ) != 0:
if constantQNorm == 'L1':
constantQ /= np.sum(np.abs(constantQ))
elif constantQNorm == 'L2':
constantQ /= np.sum(constantQ**2)
elif constantQNorm == 'Linf':
constantQ /= np.max(np.abs(constantQ))
Xtrain.append(constantQ)
# train on this pass
if len(Xtrain) > 0:
print "Xtrain: ", len(Xtrain), ", Xtarget: ", len(Xtarget)
train = np.asarray(Xtrain)
target = np.asarray(Xtarget)
trainer.setData(train, target)
trainNet(trainer, verbose)
# clear feature buffers
del Xtrain[:]
del Xtarget[:]
passInd += 1
print "pass: "******"Done training neural network."
return net
reportString += "input: " + nl + str(x) + nl
reportString += "weight layer 1: " + nl + str(w1) + nl
reportString += "bias layer 1: " + nl + str(b1) + nl
reportString += "weight layer 2: " + nl + str(w2) + nl
reportString += "bias layer 2: " + nl + str(b2) + nl
reportString += "learning rate: " + str(alpha) + nl
reportString += "target: " + nl + str(t) + nl
reportString += "epoch: " + str(epoch) + cut
for i in range(0, epoch):
for j in range(0, len(t)):
# -- Forward --
x_sample = x[j].reshape(1, x[j].shape[0])
t_sample = t[j].reshape(1, t[j].shape[0])
z = a.Sigmoid(x_sample.dot(w1) - b1)
y = a.Sigmoid(z.dot(w2) - b2)
# -- Backprop --
e2 = t_sample - y
g2 = y * (1 - y) * e2
w2_delta = alpha * np.dot(z.transpose(), g2)
b2_delta = alpha * -1.0 * g2
w2 = w2 + w2_delta
e1 = np.dot(w2, g2.transpose()).transpose()
g1 = z * (1 - z) * e1
w1_delta = alpha * np.dot(x_sample.transpose(), g1)
b1_delta = alpha * -1.0 * g1
w1 = w1 + w1_delta
def learnNNbuff(chromaNorm = 'L1', constantQNorm = None, deltaTrain = 2, nnStruct = [256, 150, 24], errorFunc = 'SSE', verbose = False):
'''
Learns neural network weights with buffered feature input (batch training in segments).
Use this function when thrashing to disk is a possibility
PARAMETERS
----------
chromaNorm: L1, L2, Linf, or None, normalization of chroma
constantQNorm: L1, L2, Linf, or None, normalization of constant q transform
deltaTrain {int}: how many songs to train after. Buffering features prevents thrashing to disk.
nnStruct {List}: neural network layer list (each element is number of neurons at the layer
errorFunc {String}: 'KLDiv' or 'SSE'
verbose {Boolean}: report progress to standard out
RETURN
------
net: trained neural network
'''
# initialize feature storage
Xtrain = [] # Constant-Q transform
Xtarget = [] # Bass and treble chromagram
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct)-2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
# hire a trainer for the network
trainer = nn.Trainer(net, errorFunc, 1)
# read constant-q transform preliminary features
qFile = open('data/logfreqspec.csv', 'r')
# read bass and treble chromagram features
cFile = open('data/bothchroma.csv', 'r')
songNum = 0
eligibleTrain = False
for cObs, qObs in izip(cFile, qFile):
cObs = cObs.split(",")
qObs = qObs.split(",")
# check if we have moved to a new song
if cObs[0]:
# check features are in sync by audio file path
if not qObs[0] or cObs[0] != qObs[0]:
raise ValueError("Feature files out of sync")
# train the neural net with buffered features from the previous songs
if songNum > 0 and songNum % deltaTrain == 0:
trainer.setData(np.asarray(Xtrain), np.asarray(Xtarget))
trainNet(trainer, verbose)
# clear feature buffers
del Xtrain[:]
del Xtarget[:]
songNum += 1
if verbose:
print "Processing song: ", cObs[0]
# double check features are in sync by timestamp
if float(cObs[1]) != float(qObs[1]):
raise ValueError("Feature files out of sync")
# get chromagrams
chroma = np.asfarray(cObs[2:])
# avoid divide by zero (this is silence in audio)
if np.sum(chroma) == 0:
continue
# perform feature normalization
if chromaNorm == 'L1':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(np.abs(chroma[12:24]))
elif chromaNorm == 'L2':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(chroma[0:12] ** 2)
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(chroma[12:24] ** 2)
elif chromaNorm == 'Linf':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.max(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.max(np.abs(chroma[12:24]))
Xtarget.append(chroma)
# get Constant-Q transform
constantQ = np.asfarray(qObs[2:])
# perform feature normalization
if constantQNorm is not None and np.sum(constantQ) != 0:
if constantQNorm == 'L1':
constantQ /= np.sum(np.abs(constantQ))
elif constantQNorm == 'L2':
constantQ /= np.sum(constantQ ** 2)
elif constantQNorm == 'Linf':
constantQ /= np.max(np.abs(constantQ))
Xtrain.append(constantQ)
# train leftovers (< deltaTrain songs)
if len(Xtrain) > 0:
trainer.setData(np.asarray(Xtrain), np.asarray(Xtarget))
trainNet(trainer, verbose)
if verbose:
print "Done training neural network."
qFile.close()
cFile.close()
return net
