def predict(theta, input_size, hidden_size, num_classes, netconfig, lambda_,
data):
# extract the part which compute the softmax gradient
softmax_theta = theta[0:hidden_size * num_classes].reshape(
num_classes, hidden_size)
# extract out the "stack"
stack = params2stack(theta[hidden_size * num_classes:], netconfig)
m = data.shape[1]
a = [data]
z = [np.array(0)]
for s in stack:
z.append(
np.dot(s['w'], a[-1]) + np.matlib.repmat(s['b'], m, 1).transpose())
a.append(sigmoid(z[-1]))
pred = softmax.predict(softmax_theta, a[-1], num_classes, hidden_size)
return pred
def main():
fnameTrainIm = sys.argv[1]
fnameTrainLb = sys.argv[2]
fnameTestIm = sys.argv[3]
fnameTestLb = sys.argv[4]
# read training data and labels
trainData, trainLabel = read(fnameTrainIm, fnameTrainLb)
# initialize constants and parameters
inputSize = 28 * 28
numClasses = 10
lam = 1e-4
DEBUG = 0
if DEBUG == 1:
# compute difference between numerical gradient and computed grad on the subset
subData, subLabel, subTheta, subPatchsize = subsample(
trainData, trainLabel, rows, numClasses)
subCost, subGrad = softmax.softmaxCost(subTheta, numClasses,
subPatchsize, lam, subData,
subLabel)
numGrad, diff = numericalgrad(subTheta, numClasses, subPatchsize, lam,
subData, subLabel, subGrad)
print "The diff is", diff
# train regression code
thetaOpt = softmax.train(numClasses, inputSize, lam, trainData, trainLabel)
# predict labels for test data
testData, testLabel = read(fnameTestIm, fnameTestLb)
pred = softmax.predict(thetaOpt, testData, numClasses, inputSize)
# report accuracy of softmax regression
accuracy = 100 * np.sum(testLabel == pred) / len(testLabel)
print "Accuracy is {0:.2f}".format(accuracy)
unlabeledData, trainData, trainLabels, testData, testLabels = GetDivideSets(
)
#step3 用无标签数据集训练自编码的特征
theta = sparseAutoencoder.initiallize(hiddenSize, inputSize)
[X,cost,d]=sop.fmin_l_bfgs_b(lambda (x) :sparseAutoencoder.sparseAutoencoderCost(x, inputSize, \
hiddenSize, la, sparsityParam, beta, unlabeledData),x0=theta,maxiter=400,disp=1)
W1 = X[0:hiddenSize * inputSize].reshape(hiddenSize, inputSize)
W1 = W1.T
opttheta = X
#step4 获得有标签训练集的激活值并用来训练softmax分类器的权值
trainImages = feedforward(opttheta, hiddenSize, inputSize, trainData)
thetaSoftmax = softmax.initiallize(numClasses, hiddenSize)
la = 1e-4
[X, cost, d] = sop.fmin_l_bfgs_b(lambda (x): softmax.SoftmaxCost(
trainImages, trainLabels, x, la, hiddenSize, numClasses),
x0=thetaSoftmax,
maxiter=100,
disp=1)
#step5 获得有标签测试集的激活值并且给出softmax分类准确率, 5分类的准确率应该在98%附近
testImages = feedforward(opttheta, hiddenSize, inputSize, testData)
optthetaSoftmax = X
accuracy = softmax.predict(optthetaSoftmax, testImages, testLabels,
hiddenSize, numClasses)
print accuracy
#画出权值图像
sparseAutoencoder.showimg(W1, hiddenSize, 28)
hiddenSize = 200
sparsityParam = 0.1
la = 3e-3
beta = 3
#step2 获取无标签数据集, 有标签训练数据集, 有标签测试数据集
unlabeledData, trainData, trainLabels, testData, testLabels = GetDivideSets()
#step3 用无标签数据集训练自编码的特征
theta = sparseAutoencoder.initiallize(hiddenSize, inputSize)
[X,cost,d]=sop.fmin_l_bfgs_b(lambda (x) :sparseAutoencoder.sparseAutoencoderCost(x, inputSize, \
hiddenSize, la, sparsityParam, beta, unlabeledData),x0=theta,maxiter=400,disp=1)
W1 = X[0:hiddenSize*inputSize].reshape(hiddenSize, inputSize)
W1 = W1.T
opttheta = X
#step4 获得有标签训练集的激活值并用来训练softmax分类器的权值
trainImages = feedforward(opttheta, hiddenSize, inputSize, trainData)
thetaSoftmax = softmax.initiallize(numClasses, hiddenSize)
la = 1e-4
[X,cost,d] = sop.fmin_l_bfgs_b(lambda (x) :softmax.SoftmaxCost(trainImages, trainLabels,x,la,hiddenSize,numClasses),x0=thetaSoftmax,maxiter=100,disp=1)
#step5 获得有标签测试集的激活值并且给出softmax分类准确率, 5分类的准确率应该在98%附近
testImages = feedforward(opttheta, hiddenSize, inputSize, testData)
optthetaSoftmax = X
accuracy = softmax.predict(optthetaSoftmax,testImages,testLabels,hiddenSize,numClasses)
print accuracy
#画出权值图像
sparseAutoencoder.showimg(W1, hiddenSize, 28)
pooledFeaturesThis = cnnPool(poolDim, convolvedFeaturesThis)
pooledFeaturesTrain[featureStart:featureEnd, :, :, :] = pooledFeaturesThis
convolvedFeaturesThis = cnnConvolve(patchDim, stepSize, testImages, Wt, bt, white, meanPatch)
pooledFeaturesThis = cnnPool(poolDim, convolvedFeaturesThis)
pooledFeaturesTest[featureStart:featureEnd, :, :, :] = pooledFeaturesThis
cnnPooledFeatures = {}
cnnPooledFeatures['pooledFeaturesTrain'] = pooledFeaturesTrain
cnnPooledFeatures['pooledFeaturesTest'] = pooledFeaturesTest
sio.savemat('cnnPooledFeatures.mat', cnnPooledFeatures)
#step7 用卷积特征来进行softmax预测,这个就很快了,十几秒, 正确率为80%左右
#Mat1 = sio.loadmat('cnnPooledFeatures.mat')
#pooledFeaturesTrain = Mat1['pooledFeaturesTrain']
#pooledFeaturesTest = Mat1['pooledFeaturesTest']
softmaxLambda = 1e-4
numClasses = 4
inputSize = int(pooledFeaturesTrain.size/numTrainImages)
print inputSize
softmaxX = np.transpose(pooledFeaturesTrain, (0, 2, 3, 1))
softmaxX = softmaxX.reshape(inputSize, numTrainImages)
softmaxY = trainLabels
theta = softmax.initiallize(numClasses, inputSize)
[X,cost,d] = sop.fmin_l_bfgs_b(lambda (x) :softmax.SoftmaxCost(softmaxX, softmaxY, x, softmaxLambda, inputSize, numClasses),x0=theta,maxiter=200,disp=1)
softmaxX = np.transpose(pooledFeaturesTest, (0, 2, 3, 1))
softmaxX = softmaxX.reshape(inputSize, numTestImages)
softmaxY = testLabels
accuracy = softmax.predict(X,softmaxX,softmaxY,inputSize,numClasses)
print accuracy
best_softmax = None
learning_rates = [1e-7, 5e-7]
regularization_strengths = [2.5e4, 5e4]
for lr in learning_rates:
for rs in regularization_strengths:
softmax = liner_classifiers_total.Softmax()
loss_history = softmax.train(X_train,
Y_train,
lr,
rs,
num_iters=1500,
batch_size=100,
verbose=True)
Y_train_pred = softmax.predict(X_train)
train_accuracy = np.mean(Y_train == Y_train_pred)
Y_val_pred = softmax.predict(X_val)
val_accuracy = np.mean(Y_val == Y_val_pred)
if val_accuracy > best_val:
best_val = val_accuracy
best_softmax = softmax
results[(lr, rs)] = train_accuracy, val_accuracy
# print out result
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' %
(lr, reg, train_accuracy, val_accuracy))
feature, label = loadData(inputfile)
feature = handleHistogram(feature)
# print(np.shape(feature), np.shape(label))
#print(feature)
k = 256
w0 = train(feature, label, k, 200000, 0.1)
saveModel("weights.txt", w0)
actual_x = [] # 绘制直线的x轴坐标
predict_x = [] # 绘制预测值的x坐标
for i in label:
actual_x.append(int(i[0]))
predict_x.append(i[0])
actual_y = actual_x # 直线的y坐标
# 得到预测值
predition = predict(feature, w0)
m, n = np.shape(predition)
error = np.mat(np.zeros((m)))
predict_y = [] # 预测值的y坐标
for i in predition:
predict_y.append(i[0])
color = np.arctan2(predict_y, predict_x)
# 绘制散点图
plt.scatter(predict_x, predict_y, s=10, c=color, alpha=1)
# 设置坐标轴范围
plt.xlim([0, 150])
plt.ylim([0, 150])
error[predict_y == actual_x] = 1
print("correct rate:", np.sum(error) / m)
plt.xlabel("actual value")
plt.ylabel("prediction")
def main():
"""
Step 0: Initialize parameter values for sparse autoencoder and softmax regression.
"""
row_size = 28
visible_size = row_size**2
num_labels = 5
hidden_size = 200
rho = 0.1
lambda_ = 3e-3
beta = 3
max_iter = 400
debug = 0
"""
Step 1: Load data from the MNIST database
Load the training set data and sort the training data into labeled and unlabeled sets.
"""
fIm = sys.argv[1]
fLb = sys.argv[2]
# read training data and labels
data, label = read(fIm, fLb)
# simulate a labeled and unlabeled set
# set the numbers that will be used for the unlabeled versus labeled sets
ixu = np.argwhere(label >= 5).flatten()
ixl = np.argwhere(label < 5).flatten()
# build a training and test set -> separate half the labeled set to use as test data
numTrain = round(ixl.shape[0] / 2) # half the examples in the set
train_val = ixl[0:numTrain]
test_val = ixl[numTrain:]
unlabeled_set = label[ixu]
unlabeled_data = data[:, ixu]
train_data = data[:, train_val]
train_set = label[train_val]
test_data = data[:, test_val]
test_set = label[test_val]
print data.shape
# output some statistics
print "# of examples in unlabeled set: {0:6d}".format(len(unlabeled_set))
print "# of examples in supervised training set: {0:6d}".format(
len(train_set))
print "# of examples in supervised test set: {0:6d}".format(len(test_set))
"""
Optional Step: Test Sparse Autoencoder
"""
if debug == 1:
subdata, sublabel, sub_visible, sub_hidden = sparseAuto.subsample(
train_data, train_set, row_size)
sub_theta = sparseAuto.initializeParameters(sub_hidden, sub_visible)
sub_grad, sub_cost = sparseAuto.costfun(sub_theta, sub_visible,
sub_hidden, lambda_, rho, beta,
subdata)
numGrad, diff = sparseAuto.numgrad(sub_theta, sub_grad, sub_visible,
sub_hidden, lambda_, rho, beta,
subdata)
print diff
"""
Step 2: Train Sparse Autoencoder
"""
print 'Step 2'
opt_theta = sparseAuto.train(visible_size, hidden_size, lambda_, rho, beta,
unlabeled_data)
"""
Step 3: Extract Features from the Supervised Data Set
"""
print 'Step 3'
train_features = sparseAuto.autoencoder(opt_theta, hidden_size,
visible_size, train_data)
test_features = sparseAuto.autoencoder(opt_theta, hidden_size,
visible_size, test_data)
"""
Step 4: Train the softmax classifier
"""
print 'Step 4'
lam = 1e-4
thetaOpt = softmax.train(num_labels, hidden_size, lam, train_features,
train_set)
"""
Step 5: Testing softmax classfier
"""
print 'Step 5'
pred = softmax.predict(thetaOpt, test_features, num_labels, hidden_size)
print pred[0:15]
print test_set[0:15]
# report accuracy of softmax regression
accuracy = 100 * np.sum(test_set == pred) / len(test_set)
print "Accuracy is {0:.2f}".format(accuracy)
def main():
"""
Step 0: Initialization
"""
# Initialize parameters for the exercise
image_dim = 64
image_channels = 3
patch_dim = 8
num_patches = 50000
visible_size = patch_dim * patch_dim * image_channels
output_size = visible_size
hidden_size = 400
epsilon = 0.1
pool_dim = 19
debug = 0
"""
Step 1: Train a sparse autoencoder (with a linear decoder)
"""
# load data from linear decoder exercise
opt_theta = np.load('opt_theta.npy')
zca_white = np.load('zca_white.npy')
mean_patch = np.load('mean_patch.npy')
# unpack W and b
W = opt_theta[0:hidden_size * visible_size].reshape(
hidden_size, visible_size)
b = opt_theta[2 * hidden_size *
visible_size:2 * hidden_size * visible_size + hidden_size]
"""
Step 2a: Implement convolution
"""
# read in train data from mat file
mat_contents = sio.loadmat('stlTrainSubset.mat')
train_images = mat_contents['trainImages']
train_labels = mat_contents['trainLabels']
num_train_images = mat_contents['numTrainImages'][0][0]
# read in test data from mat file
mat_contents = sio.loadmat('stlTestSubset.mat')
test_images = mat_contents['testImages']
test_labels = mat_contents['testLabels']
num_test_images = mat_contents['numTestImages'][0][0]
# use only the first 8 images for testing
conv_images = train_images[:, :, :, 0:8]
# use only the first 8 images to test
convolved_features = cnn.convolve(patch_dim, hidden_size, conv_images, W,
b, zca_white, mean_patch)
if debug == 1:
"""
Step 2b: Check your convolution
"""
for i in range(1000):
feature_num = np.random.randint(0, hidden_size)
image_num = np.random.randint(0, 8)
image_row = np.random.randint(0, image_dim - patch_dim + 1)
image_col = np.random.randint(0, image_dim - patch_dim + 1)
patch = conv_images[image_row:image_row + patch_dim,
image_col:image_col + patch_dim, :, image_num]
patch = np.concatenate(
(patch[:, :,
0].flatten(), patch[:, :,
1].flatten(), patch[:, :,
2].flatten()))
patch = np.reshape(patch, (patch.size, 1))
patch = patch - np.tile(mean_patch,
(patch.shape[1], 1)).transpose()
patch = zca_white.dot(patch)
features = sparseAuto.autoencoder(opt_theta, hidden_size,
visible_size, patch)
if abs(features[feature_num, 0] -
convolved_features[feature_num, image_num, image_row,
image_col]) > 1e-9:
print 'convolved feature does not activation from autoencoder'
print 'feature number:', feature_num
print 'image number:', image_num
print 'image row:', image_row
print 'image column:', image_col
print 'convolved feature:', convolved_features[feature_num,
image_num,
image_row,
image_col]
print 'sparse AE feature:', features[feature_num, 0]
sys.exit(
'convolved feature does not match activation from autoencoder'
)
print('congrats! convolution code passed the test')
"""
Step 2c: Implement Pooling
"""
# pooled_features = cnn.pooling(pool_dim, convolved_features)
"""
Step 2d: Checking your pooling
"""
test_matrix = np.arange(64).reshape(8, 8)
expected_matrix = np.array(
[[np.mean(test_matrix[0:4, 0:4]),
np.mean(test_matrix[0:4, 4:8])],
[np.mean(test_matrix[4:8, 0:4]),
np.mean(test_matrix[4:8, 4:8])]])
test_matrix = test_matrix.reshape(1, 1, 8, 8)
pooled_features = cnn.pooling(4, test_matrix)
if not (pooled_features == expected_matrix).all():
print 'pooling incorrect'
print 'expected:'
pprint.pprint(expected_matrix)
print 'got:'
pprint.pprint(pooled_features)
else:
print "congrats! pooling code passed the test"
"""
Step 3: Convolve and pool with the data set
"""
step_size = 50
assert hidden_size % step_size == 0
pooled_features_train = np.zeros(
(hidden_size, num_train_images,
int(np.floor((image_dim - patch_dim + 1) / pool_dim)),
int(np.floor((image_dim - patch_dim + 1) / pool_dim))))
pooled_features_test = np.zeros(
(hidden_size, num_test_images,
int(np.floor((image_dim - patch_dim + 1) / pool_dim)),
int(np.floor((image_dim - patch_dim + 1) / pool_dim))))
start_time = time.time()
for conv_part in range(hidden_size / step_size):
feature_start = conv_part * step_size
feature_end = (conv_part + 1) * step_size
print 'Step:', conv_part, 'Features', feature_start, 'to', feature_end
Wt = W[feature_start:feature_end, :]
bt = b[feature_start:feature_end]
print 'Convolving and pooling train images'
convolved_features_this = cnn.convolve(patch_dim, step_size,
train_images, Wt, bt, zca_white,
mean_patch)
pooled_features_this = cnn.pooling(pool_dim, convolved_features_this)
pooled_features_train[
feature_start:feature_end, :, :, :] = pooled_features_this
print 'Elapsed time is', str(
datetime.timedelta(seconds=time.time() - start_time))
print 'Convolving and pooling test images'
convolved_features_this = cnn.convolve(patch_dim, step_size,
test_images, Wt, bt, zca_white,
mean_patch)
pooled_features_this = cnn.pooling(pool_dim, convolved_features_this)
pooled_features_test[
feature_start:feature_end, :, :, :] = pooled_features_this
print 'Elapsed time is', str(
datetime.timedelta(seconds=time.time() - start_time))
np.save('pooled_features_train.npy', pooled_features_train)
np.save('pooled_features_test.npy', pooled_features_test)
print 'Elapsed time is', str(
datetime.timedelta(seconds=time.time() - start_time))
"""
Step 4: Use pooled features for classification
"""
# set up parameters for softmax
softmax_lambda = 1e-4
num_classes = 4
# reshape the pooled_features to form an input vector for softmax
softmax_x = np.transpose(pooled_features_train, [0, 2, 3, 1])
softmax_x = softmax_x.reshape(
(pooled_features_train.size / num_train_images, num_train_images))
softmax_y = train_labels.flatten() - 1
softmax_opt_theta = softmax.train(
num_classes, pooled_features_train.size / num_train_images,
softmax_lambda, softmax_x, softmax_y)
np.save('theta_opt_theta.npy', opt_theta)
"""
Step 5: Test Classifier
"""
softmax_x = np.transpose(pooled_features_test, [0, 2, 3, 1])
softmax_x = softmax_x.reshape(
(pooled_features_test.size / num_test_images, num_test_images))
softmax_y = test_labels.flatten() - 1
pred = softmax.predict(softmax_opt_theta, softmax_x, num_classes,
pooled_features_train.size / num_train_images)
accuracy = 100 * np.sum(softmax_y == pred) / len(softmax_y)
print "Accuracy is {0:.2f}".format(accuracy)
softmax = Softmax() #创建对象，此时W为空
tic = time.time()
loss_hist = softmax.train(x_train,
y_train,
learning_rate=1e-7,
reg=2.5e4,
num_iters=1500,
verbose=True) #此时svm对象中有W
toc = time.time()
print('that took %fs' % (toc - tic))
plt.plot(loss_hist)
plt.xlabel('iteration number')
plt.ylabel('loss value')
plt.show()
#训练完成之后，将参数保存，使用参数进行预测，计算准确率
y_train_pred = softmax.predict(x_train)
print('training accuracy: %f' % (np.mean(y_train == y_train_pred))) #返回条件成立的占比
y_val_pred = softmax.predict(x_val)
print('validation accuracy: %f' % (np.mean(y_val == y_val_pred)))
#在拿到一组数据时一般分为训练集，开发集（验证集），测试集。训练和测试集都知道是干吗的，验证集在除了做验证训练结果外
# 还可以做超参数调优，寻找最优模型。遍历每一种参数组合，训练Softmax模型，然后在验证集上测试，寻找验证集上准确率最高的模型
# 交叉验证很费时间，下面的代码总共循环18次，在9700KCPU上满载运行了十多分钟，自己选择是否运行该段代码。
# 超参数调优（交叉验证）
learning_rates = [1.4e-7, 1.5e-7, 1.6e-7]
# for循环的简化写法12个
regularization_strengths = [(1 + i * 0.1) * 1e4
for i in range(-3, 3)] + [(2 + i * 0.1) * 1e4
for i in range(-3, 3)]
results = {} # 字典
def softmaxCostCallback(x):
return softmax.cost(x, numClasses, inputSize, lambdaParam, trainData,
trainLabels)
# Randomly initialise theta
thetaParam = 0.005 * random.normal(size=numClasses * inputSize)
options = {
'maxiter': 100,
'disp': True,
}
result = optimize.minimize(softmaxCostCallback,
thetaParam,
method='L-BFGS-B',
jac=True,
options=options)
optTheta = result.x[0:numClasses * inputSize].reshape(numClasses, inputSize)
# Evaluating performance of the softmax classifier
testData = loadMNISTImages('mnist/t10k-images-idx3-ubyte')
testLabels = loadMNISTLabels('mnist/t10k-labels-idx1-ubyte')
pred = softmax.predict(optTheta, testData)
acc = mean(testLabels == pred)
print('Accuracy: %0.3f%%\n' % (acc * 100))
inputSize = 28 * 28 # Size of input vector (MNIST images are 28x28)
numClasses = 10 # Number of classes (MNIST images fall into 10 classes)
lambdaParam = 1e-4 # Weight decay parameter
trainData = loadMNISTImages('mnist/train-images-idx3-ubyte')
trainLabels = loadMNISTLabels('mnist/train-labels-idx1-ubyte')
def softmaxCostCallback(x):
return softmax.cost(x, numClasses, inputSize, lambdaParam, trainData, trainLabels)
# Randomly initialise theta
thetaParam = 0.005 * random.normal(size=numClasses * inputSize)
options = {
'maxiter': 100,
'disp': True,
}
result = optimize.minimize(softmaxCostCallback, thetaParam, method='L-BFGS-B', jac=True, options=options)
optTheta = result.x[0:numClasses*inputSize].reshape(numClasses, inputSize)
# Evaluating performance of the softmax classifier
testData = loadMNISTImages('mnist/t10k-images-idx3-ubyte')
testLabels = loadMNISTLabels('mnist/t10k-labels-idx1-ubyte')
pred = softmax.predict(optTheta, testData)
acc = mean(testLabels==pred)
print('Accuracy: %0.3f%%\n' % (acc * 100))
featureStart:featureEnd, :, :, :] = pooledFeaturesThis
cnnPooledFeatures = {}
cnnPooledFeatures['pooledFeaturesTrain'] = pooledFeaturesTrain
cnnPooledFeatures['pooledFeaturesTest'] = pooledFeaturesTest
sio.savemat('cnnPooledFeatures.mat', cnnPooledFeatures)
#step7 用卷积特征来进行softmax预测,这个就很快了,十几秒, 正确率为80%左右
#Mat1 = sio.loadmat('cnnPooledFeatures.mat')
#pooledFeaturesTrain = Mat1['pooledFeaturesTrain']
#pooledFeaturesTest = Mat1['pooledFeaturesTest']
softmaxLambda = 1e-4
numClasses = 4
inputSize = int(pooledFeaturesTrain.size / numTrainImages)
print inputSize
softmaxX = np.transpose(pooledFeaturesTrain, (0, 2, 3, 1))
softmaxX = softmaxX.reshape(inputSize, numTrainImages)
softmaxY = trainLabels
theta = softmax.initiallize(numClasses, inputSize)
[X, cost, d] = sop.fmin_l_bfgs_b(lambda (x): softmax.SoftmaxCost(
softmaxX, softmaxY, x, softmaxLambda, inputSize, numClasses),
x0=theta,
maxiter=200,
disp=1)
softmaxX = np.transpose(pooledFeaturesTest, (0, 2, 3, 1))
softmaxX = softmaxX.reshape(inputSize, numTestImages)
softmaxY = testLabels
accuracy = softmax.predict(X, softmaxX, softmaxY, inputSize, numClasses)
print accuracy