def main():
"""
程序入口
"""
# 加载数据
train_X, train_Y, test_X, test_Y = utils.load_data_sets()
print "1. show the data set"
plt.scatter(test_X.T[:, 0], test_X.T[:, 1], c=test_Y, s=40,
cmap=plt.cm.Spectral)
plt.title("show the data set")
plt.show()
print "2. begin to training"
# 训练模型
parameters = train(train_X, train_Y, n_h=10, num_iterations=10000,
print_cost=True)
# 预测训练集
predictions = predict(parameters, train_X)
# 输出准确率
print('Train Accuracy: %d' % float((np.dot(train_Y, predictions.T) +
np.dot(1 - train_Y, 1 - predictions.T)) /
float(train_Y.size) * 100) + '%')
# 预测测试集
predictions = predict(parameters, test_X)
print('Test Accuracy: %d' % float((np.dot(test_Y, predictions.T) +
np.dot(1 - test_Y, 1 - predictions.T)) /
float(test_Y.size) * 100) + '%')
# Plot the decision boundary
print "3. output the division"
utils.plot_decision_boundary(lambda x: predict(parameters, x.T), train_X,
train_Y)
plt.title("Decision Boundary for hidden layer size " + str(4))
plt.show()
def bagging_pasting(X_train, X_val, y_train, y_val, doplot=False):
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
tree_clf = DecisionTreeClassifier(random_state=42)
tree_clf.fit(X_train, y_train)
y_pred_tree = tree_clf.predict(X_val)
print("DecisionTree classifier, accuracy score = %f\n" %
accuracy_score(y_val, y_pred_tree))
bag_clf = BaggingClassifier(base_estimator=DecisionTreeClassifier(),
n_estimators=500,
max_samples=100,
bootstrap=True,
n_jobs=1,
oob_score=True)
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_val)
print("Bagging classifier, accuracy score = %f\n" %
accuracy_score(y_val, y_pred))
if doplot:
from utils import plot_decision_boundary, save_fig
plt.figure(figsize=(11, 4))
plt.subplot(121)
plot_decision_boundary(tree_clf, X, y)
plt.title("Decision Tree", fontsize=14)
plt.subplot(122)
plot_decision_boundary(bag_clf, X, y)
plt.title("Decision Trees with Bagging", fontsize=14)
save_fig("DT_without_and_with_bagging_plot", CHAPTER_ID)
plt.show()
def run():
# Generate a dataset and plot it
np.random.seed(0)
X, y = sklearn.datasets.make_moons(200, noise=0.20)
utils.plot_training_examples(X, y)
layers_dim = [2, 3, 2]
model = Sequential(layers_dim)
model.train(X, y, epoches=20000, learning_rate=0.01, print_metrics=True)
utils.plot_decision_boundary(lambda x: model.predict(x), X, y)
utils.plot_show()
def evaluate_dataset(X, Y):
# Print the shapes of input data
shape_X = X.shape
shape_Y = Y.shape
m = shape_X[1] # training set size
print('The shape of X is: ' + str(X.shape))
print('The shape of Y is: ' + str(Y.shape))
print(f'I have m = {m} training examples')
# Visualize the data:
Y_flat = Y.ravel()
plt.scatter(X[0,:], X[1,:], c=Y_flat, s=40, cmap=plt.cm.Spectral)
plt.show()
# Train the logistic regression classifier
clf = sklearn.linear_model.LogisticRegressionCV();
clf.fit(X.T, Y_flat);
# Plot the decision boundary for logistic regression
plot_decision_boundary(lambda x: clf.predict(x), X, Y)
plt.title("Logistic Regression")
# Print accuracy
LR_predictions = clf.predict(X.T)
print(f'Accuracy of logistic regression: {float((np.dot(Y,LR_predictions) + np.dot(1-Y,1-LR_predictions))/float(Y.size)*100):.4f}%')
plt.show()
###
# Build a model with a n_h-dimensional hidden layer
dimensions = [X.shape[0], 20, 7, 5, Y.shape[0]]
activations = ['', ACTIVATION_FUNCTION_RELU, ACTIVATION_FUNCTION_RELU, ACTIVATION_FUNCTION_RELU, ACTIVATION_FUNCTION_SIGMOID]
# hyperparams = { HYPERPARAM_LEARNING_RATE: 0.3, HYPERPARAM_LEARNING_STEPS: 10000, HYPERPARAM_DROPOUT_KEEP_PROB: 0.86 }
hyperparams = { HYPERPARAM_LEARNING_RATE: 0.0007, HYPERPARAM_LEARNING_STEPS: 10000, HYPERPARAM_MINI_BATCH_SIZE: 64, HYPERPARAM_MOMENTUM_RATE: 0.9, HYPERPARAM_ADAM_RATE: 0.999 }
model, costs = train_model(dimensions, activations, hyperparams, X, Y)
pred = forward_pass(model, X) > 0.5
# Plot the decision boundary
plot_decision_boundary(lambda x: forward_pass(model, x.T) > 0.5, X, Y)
plt.title('Decision Boundary for NN')
# Print accuracy
predictions = forward_pass(model, X)
print(f'Accuracy: {float((np.dot(Y,predictions.T) + np.dot(1-Y,1-predictions.T))/float(Y.size)*100):.4f}%')
plt.show()
def train(self, X_test, y_test, num_passes=100):
X_train = self.X_train
num_classes = len(self.get_classes())
encoder = LabelEncoder()
y_train = encoder.fit_transform(self.y_train)
y_test = encoder.transform(y_test)
y_p = np.zeros((len(y_train), num_classes))
y_p[np.arange(len(y_train)), y_train] = 1
model_bb = perceptron.build_model(self.hidden_nodes, X_train, y_p,
num_passes=num_passes, epsilon=1e-2,
epoch=100,
eps_factor=1.0,
print_loss=True, print_epoch=10,
batch_size=20,
force_reg=self.force_reg)
perceptron.save(model_bb, 'experiments/{}/models/oracle_{}.pkl'.
format(self.dataset, self.hidden_nodes))
y_pred = perceptron.predict(model_bb, X_test)
print Counter(y_pred)
acc = accuracy_score(y_test, y_pred)
print 'Training accuracy: {}'.format(acc)
if X_train.shape[1] == 2 and plt is not None:
bounds = [-1.1, 1.1, -1.1, 1.1]
X_train = X_train.values
plt.figure()
plt.scatter(X_train[:, 0], X_train[:, 1], s=40, c=y_train,
cmap=plt.cm.Spectral)
plt.savefig('experiments/{}/plots/data_{}'.
format(self.dataset, len(X_train)))
filename = 'experiments/{}/plots/oracle_{}_boundary'.\
format(self.dataset, self.hidden_nodes)
utils.plot_decision_boundary(lambda x:
perceptron.predict(model_bb, x),
X_train, y_train, bounds, filename)
def adaboost(X_train, X_val, y_train, y_val, doplot=False):
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
ada_clf = AdaBoostClassifier(
base_estimator=DecisionTreeClassifier(max_depth=1), # weak classifier
n_estimators=200,
algorithm='SAMME.R',
learning_rate=0.5)
ada_clf.fit(X_train, y_train)
y_pred = ada_clf.predict(X_val)
print("Adaboost classifier, accuracy score = %f\n" %
accuracy_score(y_val, y_pred))
if doplot:
from utils import plot_decision_boundary, save_fig
plt.figure(figsize=(11, 4))
plot_decision_boundary(ada_clf, X, y)
plt.title("Adaboost classifcation with Decision Tree base estimator",
fontsize=12)
save_fig("AdaBoost_with_DT", CHAPTER_ID)
plt.show()
def main():
"""
show dataset and the result of logistic regression
"""
np.random.seed(1)
# 加载数据
train_X, train_Y, test_X, test_Y = utils.load_data_sets()
print "1. show the data set"
plt.scatter(test_X.T[:, 0], test_X.T[:, 1], c=test_Y, s=40,
cmap=plt.cm.Spectral)
plt.title("show the data set")
plt.show()
shape_X = train_X.shape
shape_Y = train_Y.shape
m = train_Y.shape[1]
print 'The shape of X is: ' + str(shape_X)
print 'The shape of Y is: ' + str(shape_Y)
print 'I have m = %d training examples!' % (m)
clf = sklearn.linear_model.LogisticRegressionCV()
clf.fit(train_X.T, train_Y.T)
print "2. show the result of logistic classification"
utils.plot_decision_boundary(lambda x: clf.predict(x), train_X, train_Y)
plt.title("Logistic Regression")
plt.show()
lr_predictions = clf.predict(train_X.T)
print ('Accuracy of logistic regression: %d ' % float((np.dot(train_Y, lr_predictions)
+ np.dot(1 - train_Y, 1 - lr_predictions)) / float(train_Y.size) * 100)
+ '% ' + "(percentage of correctly labelled datapoints)")
X = tf.cast(X, tf.float64)
A2, cache = forward_propagation(X, parameters)
predictions = (A2 > 0.5)
predictions = np.array(predictions, dtype=np.bool)
return predictions
if __name__ == "__main__":
X, Y = load_planar_dataset()
parameters = nn_model(X, Y, n_h=4, num_iterations=10000, print_cost = True)
#plot the decision boundary
plot_decision_boundary(lambda x: predict(parameters, x.T), X, Y)
plt.title("Decision boundary for hidden layer size " + str(4))
# Accuracy
predictions = predict(parameters, X)
print(f'Accuracy: {float((np.dot(Y, predictions.T) + np.dot(1-Y, 1-predictions.T))/float(Y.size)* 100)} % ')
# Tunung different size of hidden layer
plt.figure(figsize=(16, 32))
hidden_layer_size = [1, 2, 3, 4, 5, 20, 50]
for i, n_h, in enumerate(hidden_layer_size):
plt.subplot(5, 2, i+1)
plt.title('Hidden layer of size %d' %n_h)
parameters = nn_model(X, Y, n_h, num_iterations = 5000)
plot_decision_boundary(lambda x: predict(parameters, x.T), X, Y)
np.random.seed(3)
train_X, train_Y = sklearn.datasets.make_moons(n_samples=300, noise=.2)
train_X = train_X.T
train_Y = train_Y.reshape((1, train_Y.shape[0]))
return train_X, train_Y
if __name__ == '__main__':
train_X, train_Y = load_dataset()
layer_dims = [train_X.shape[0], 5, 2, 1]
dnn = DNN(X=train_X,
Y=train_Y,
layer_dims=layer_dims,
epochs=10000,
alpha=0.0007,
print_loss=True,
print_loss_iter=1000,
initialization="he",
optimizer='momentum')
dnn.fit()
# dnn.fit_regularization()
# dnn.fit_dropout()
accuracy = dnn.score(train_X, train_Y)
print('train:', accuracy)
# print('test', dnn.score(test_X, test_Y))
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: dnn.predict(x.T), train_X, train_Y,
'momentum')
W += -learning_rate * dW
b += -learning_rate * db
##======================================================================
## STEP 4: Testing on training data
#
# You should now test your model against the training examples.
# To do this, you will first need to write softmaxPredict
# (in softmax.py), which should return predictions
# given a softmax model and the input data.
pred = softmaxPredict(W, b, instances)
acc = np.mean(labels == pred)
print('Accuracy: %0.3f%%.' % (acc * 100))
# Accuracy is the proportion of correctly classified images
# After 200 epochs, the results for our implementation were:
#
# Spiral Accuracy: 53.3%
# Flower Accuracy: 47.0%
##======================================================================
## STEP 5: Plot the decision boundary.
#
utils.plot_decision_boundary(lambda x: softmaxPredict(W, b, x), instances,
labels)
plt.title("Softmax Regression")
plt.savefig(FLAGS.input_data + '-boundary.jpg')
plt.show()
model = {"W1": W1, "b1": b1, "W2": W2, "b2": b2}
for i in range(num_passess):
W1, b1, W2, b2, a1, a2 = forward(model, X)
dW1, db1, dW2, db2 = backpropagate(X, W2, a1, a2)
# regularize
dW2 += reg_lambda * W2
dW1 += reg_lambda * W1
# update
model['W1'] += -epsilon * dW1
model['b1'] += -epsilon * db1
model['W2'] += -epsilon * dW2
model['b2'] += -epsilon * db2
if print_loss and i % 1000 == 0:
print("Loss after iteration {}: {}".format(
i, calculate_loss(model, X)))
return model
if __name__ == '__main__':
nn_input_dim = 2
nn_output_dim = 2
epsilon = 0.01
reg_lambda = 0.01
nn_hdim = 3
model = build_model(X, epsilon, reg_lambda, nn_hdim, print_loss=True)
utils.plot_decision_boundary(X, y, lambda x: predict(model, x))
test_X = data['Xval'].T
test_Y = data['yval'].T
# plt.scatter(train_X[0, :], train_X[1, :], c=train_Y.ravel(), s=40, cmap=plt.cm.Spectral)
return train_X, train_Y, test_X, test_Y
if __name__ == '__main__':
train_X, train_Y, test_X, test_Y = load_2D_dataset()
layer_dims = [train_X.shape[0], 20, 3, 1]
dnn = DNN(X=train_X,
Y=train_Y,
layer_dims=layer_dims,
max_iter=30000,
alpha=0.3,
print_loss=True,
print_loss_iter=10000,
lambd=0.7,
keep_prob=0.86)
# dnn.fit()
# dnn.fit_regularization()
dnn.fit_dropout()
accuracy = dnn.score(train_X, train_Y)
print('train:', accuracy)
print('test', dnn.score(test_X, test_Y))
axes = plt.gca()
axes.set_xlim([-0.75, 0.40])
axes.set_ylim([-0.75, 0.65])
plot_decision_boundary(lambda x: dnn.predict(x.T), train_X, train_Y,
'Model dropout')
metrics=['binary_accuracy'])
# Create directory and callbacks to save model+checkpoints
params_path = os.path.join(directory, "good_model_params")
pickle_out = open(params_path, "wb")
pickle.dump(good_model_params, pickle_out)
filepath = os.path.join(directory, "good_model_{epoch}.hdf5")
checkpoint = keras.callbacks.ModelCheckpoint(filepath,
verbose=0,
save_best_only=False,
save_weights_only=True)
tensorboard_callback = keras.callbacks.TensorBoard(histogram_freq=1)
callbacks_list = [checkpoint, tensorboard_callback]
# Train
if not save_weights:
callbacks_list = None
model.fit(train_data,
train_label,
epochs=2000,
batch_size=32,
verbose=0,
validation_split=0.1,
shuffle=True,
callbacks=callbacks_list)
score = model.evaluate(test_data, test_label, batch_size=32)
print(f'The evaluation accuracy on the good model is: {score[1]}')
print(f'The loss value on the good model is: {score[0]}\n')
# Plot the decision boundary
plot_decision_boundary(model, filename='good_model_decision_boundary')
def main(experiment_path):
ins, outs = build_graph(lr=1e-1, w=0.5, tw=0.5)
s_size = 6
u_size = s_size
x_s, y_s = sklearn.datasets.make_moons(s_size, noise=0.1)
x_u, _ = sklearn.datasets.make_moons(2000, noise=0.1)
x_val, y_val = sklearn.datasets.make_moons(2000, noise=0.1)
writer = SummaryWriter(experiment_path)
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
for i in tqdm(range(1, 100000 + 1)):
loss_stu, loss_tea, _, _ = sess.run(
[
outs["loss_student"],
outs["loss_teacher"],
outs["student_update"],
outs["teacher_update"],
],
feed_dict={
ins["x_u"]: x_u[np.random.choice(x_u.shape[0], u_size, replace=False)],
ins["x_s"]: x_s,
ins["y_s"]: y_s,
},
)
if i % 100 == 0:
acc_tea, acc_stu = sess.run(
[outs["teacher"]["accuracy"], outs["student"]["accuracy"]],
feed_dict={
ins["x_s"]: x_val,
ins["y_s"]: y_val,
},
)
writer.add_scalar("teacher/loss", loss_tea, global_step=i)
writer.add_scalar("student/loss", loss_stu, global_step=i)
writer.add_scalar("teacher/accuracy", acc_tea, global_step=i)
writer.add_scalar("student/accuracy", acc_stu, global_step=i)
fig = plot_decision_boundary(
x_s,
y_s,
lambda x: sess.run(outs["teacher"]["class_index"], feed_dict={ins["x_s"]: x}),
)
writer.add_figure("teacher/sup", fig, global_step=i)
fig = plot_decision_boundary(
x_val,
y_val,
lambda x: sess.run(outs["teacher"]["class_index"], feed_dict={ins["x_s"]: x}),
)
writer.add_figure("teacher/val", fig, global_step=i)
fig = plot_decision_boundary(
x_s,
y_s,
lambda x: sess.run(outs["student"]["class_index"], feed_dict={ins["x_s"]: x}),
)
writer.add_figure("student/sup", fig, global_step=i)
fig = plot_decision_boundary(
x_val,
y_val,
lambda x: sess.run(outs["student"]["class_index"], feed_dict={ins["x_s"]: x}),
)
writer.add_figure("student/val", fig, global_step=i)
writer.flush()
# -----------------------------he init-----------------------------
dnn3 = DeepNeuralNetwork(layers_dims, activations, init='he')
dnn3 = dnn3.fit(train_X, train_Y, 15000, 0.01, print_cost=False, print_num=1000)
predictions_train = dnn3.accuracy_score(train_X, train_Y)
print ("he init accuracy on train: {:.2f}".format(predictions_train))
predictions_test = dnn3.accuracy_score(test_X, test_Y)
print ("he init accuracy on test: {:.2f}".format(predictions_test))
# plot decision boundary for zero initialization
plt.title("Model with zero initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: dnn1.predict(x.T), train_X, train_Y)
# plot decision boundary for large random initialization
plt.title("Model with large random initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: dnn2.predict(x.T), train_X, train_Y)
# plot decision boundary for he initialization
plt.title("Model with he initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: dnn3.predict(x.T), train_X, train_Y)
def train(self, num_repr, X_test, y_test):
X_train = self.X_train
y_train = self.y_train
y_train_p = np.zeros((len(y_train), len(self.classes)))
y_train_p[np.arange(len(y_train)), y_train] = 1
"""
assert num_repr >= len(self.get_classes())
X_repr_bb = []
class_counter = Counter(y_train)
repr_per_class \
= (num_repr + len(self.get_classes()) - 1) / len(self.get_classes())
for (c, count) in sorted(class_counter.iteritems(), key=lambda _: _[1]):
print c, count, repr_per_class
reprs = X_train.values[np.where(y_train == c)[0][0:repr_per_class]]
X_repr_bb.append(reprs)
X_repr_bb = np.vstack(X_repr_bb)[0:num_repr]
print '{} representers'.format(len(X_repr_bb))
"""
X_repr_bb = X_train[0:num_repr].values
"""
import math
import matplotlib.pyplot as plt
side = math.sqrt(X_repr_bb.shape[1])
plt.figure()
for i in range(len(X_repr_bb)):
plt.subplot(2, len(X_repr_bb)/2, i + 1)
plt.axis('off')
image = X_repr_bb[i, :].reshape((side, side))
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
plt.show()
"""
"""
model_bb = krs.build_model(X_repr_bb, False, X_train, y_train_p,
epsilon=epsilon, reg_lambda=1e-4,
num_passes=num_passes, eps_factor=0.99,
epoch=100, print_epoch=10, batch_size=1,
gamma=gamma)
y_pred_bb = krs.predict(model_bb, X_test)
"""
best_model = None
best_acc = 0
best_gamma = None
for gamma in [10**x for x in range(-10, 10)]:
from sklearn.metrics.pairwise import rbf_kernel
from sklearn.linear_model import LogisticRegression
X_train_ker = rbf_kernel(X_train, X_repr_bb, gamma=gamma)
model = LogisticRegression(multi_class='multinomial',
solver='lbfgs').\
fit(X_train_ker, y_train)
y_pred = model.predict(rbf_kernel(X_test, X_repr_bb, gamma=gamma))
acc = accuracy_score(y_test, y_pred)
if acc > best_acc:
best_acc = acc
best_gamma = gamma
best_model = model
W = best_model.coef_.T
b = best_model.intercept_
if len(self.classes) == 2:
W = np.hstack((np.zeros((len(X_repr_bb), 1)), W))
b = np.hstack((0, b))
W = theano.shared(
value=W,
name='W',
borrow=True
)
b = theano.shared(
value=b,
name='b',
borrow=True
)
model_bb = krs.KernelLog(T.matrix('x'), best_gamma, len(self.classes),
X_repr_bb, learn_kernel_base=False, W=W, b=b)
y_pred_bb = krs.predict(model_bb, X_test)
print 'Best Gamma: {}'.format(best_gamma)
print 'Y_test: {}'.format(Counter(y_test))
print 'Y_pred: {}'.format(Counter(y_pred_bb))
acc = accuracy_score(y_test, y_pred_bb)
print 'Training accuracy: {}'.format(acc)
print >> sys.stderr, 'Training accuracy: {}'.format(acc)
X_test_u = utils.gen_query_set(X_test.shape[1], 10000)
y_pred_u = krs.predict(model_bb, X_test_u)
print 'Y_pred_u: {}'.format(Counter(y_pred_u))
if X_train.shape[1] == 2:
bounds = [-1.1, 1.1, -1.1, 1.1]
X_train = X_train.values
utils.plot_decision_boundary(
lambda x: krs.predict(model_bb, x), X_train, y_train, bounds)
krs.save(model_bb,
'experiments/KLR/{}/models/oracle.pkl'.format(self.dataset))
def load_dataset():
np.random.seed(1)
train_X, train_Y = sklearn.datasets.make_circles(n_samples=300, noise=.05)
np.random.seed(2)
test_X, test_Y = sklearn.datasets.make_circles(n_samples=100, noise=.05)
# Visualize the data
# plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y, s=40, cmap=plt.cm.Spectral)
train_X = train_X.T
train_Y = train_Y.reshape((1, train_Y.shape[0]))
test_X = test_X.T
test_Y = test_Y.reshape((1, test_Y.shape[0]))
return train_X, train_Y, test_X, test_Y
if __name__ == '__main__':
train_X, train_Y, test_X, test_Y = load_dataset()
layer_dims = [train_X.shape[0], 10, 5, 1]
dnn = DNN(X=train_X, Y=train_Y, layer_dims=layer_dims, max_iter=15000, alpha=0.01,
print_loss=True, print_loss_iter=1000, lambd=0.7, keep_prob=0.86, initialization="he")
dnn.fit()
# dnn.fit_regularization()
# dnn.fit_dropout()
accuracy = dnn.score(train_X, train_Y)
print('train:', accuracy)
print('test', dnn.score(test_X, test_Y))
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: dnn.predict(x.T), train_X, train_Y, 'my random init')
plt.scatter(x[y == 1, 0], x[y == 1, 1])
plt.show()
# 使用多项式特征的SVM
def polynomial_svc(degree, C=1.0):
return Pipeline([('poly', PolynomialFeatures(degree=degree)),
('std_scaler', StandardScaler()),
('linearSVC', LinearSVC(C=C))])
poly_svc = polynomial_svc(degree=3)
poly_svc.fit(x, y)
utils.plot_decision_boundary(poly_svc, axis=[-1.5, 2.5, -1, 1.5])
plt.scatter(x[y == 0, 0], x[y == 0, 1])
plt.scatter(x[y == 1, 0], x[y == 1, 1])
plt.show()
# 使用多项式核函数SVM
def polynomial_kernel_svc(degree, C=1.0):
return Pipeline([('std_scaler', StandardScaler()),
('kernelSVC', SVC(kernel='poly', degree=degree, C=C))])
poly_kernel_svc = polynomial_kernel_svc(degree=3)
poly_kernel_svc.fit(x, y)
def extract(self,
X_train,
y_train,
num_repr,
budget,
steps=[],
adaptive_oracle=False,
baseline=False,
gamma=1,
epsilon=1e-2,
reg_lambda=1e-16,
eps_factor=0.99,
epoch=100,
print_epoch=10,
batch_size=1,
num_passes=1000,
random_seed=0):
numpy.random.seed(random_seed)
assert not (adaptive_oracle and steps)
if steps:
step = steps[0]
else:
step = budget
if not adaptive_oracle:
X_ext = utils.gen_query_set(self.num_features(), step)
else:
X_ext = utils.line_search_oracle(self.num_features(), step,
self.query)
X_repr = utils.gen_query_set(self.num_features(), num_repr)
model = None
idx = 0
while budget > 0:
idx += 1
budget -= step
y_ext = self.query(X_ext)
if not baseline:
y_ext_p = self.query_probas(X_ext)
else:
num_classes = len(self.get_classes())
y_ext_p = numpy.zeros((len(y_ext), num_classes))
y_ext_p[numpy.arange(len(y_ext)), y_ext] = 1
print y_ext_p
print '{} ({})'.format(
self.calculate_loss(X_ext, y_ext_p, reg_lambda),
self.calculate_loss(X_ext, y_ext_p, 0))
print >> sys.stderr, self.calculate_loss(X_ext, y_ext_p,
reg_lambda)
model = build_model(X_repr,
False,
X_ext,
y_ext_p,
gamma=gamma,
epsilon=epsilon,
reg_lambda=reg_lambda,
num_passes=num_passes,
eps_factor=eps_factor,
epoch=epoch,
print_epoch=print_epoch,
batch_size=batch_size,
warm_start=model)
mtype = "base" if baseline else "extr"
mode = "adapt-local" if steps \
else "adapt-oracle" if adaptive_oracle \
else "passive"
save(
model, 'experiments/KLR/{}/models/{}_{}_{}.pkl'.format(
self.dataset, mode, mtype, len(X_ext)))
if X_train is not None and X_train.shape[1] == 2:
bounds = [-1.1, 1.1, -1.1, 1.1]
filename = 'experiments/KLR/{}/plots/{}_{}_{}_{}_boundary'.\
format(self.dataset, mode, mtype, len(X_ext), random_seed)
utils.plot_decision_boundary(lambda x: predict(model, x),
X_train.values, y_train, bounds,
filename)
filename = 'experiments/KLR/{}/plots/{}_{}_{}_{}_boundary_ext'.\
format(self.dataset, mode, mtype, len(X_ext), random_seed)
utils.plot_decision_boundary(lambda x: predict(model, x),
X_ext, y_ext, bounds, filename)
if budget > 0:
step = steps[idx] - steps[idx - 1]
X_local = utils.gen_query_set(n=X_repr.shape[1],
test_size=1000)
Y_local = predict(model, X_local)
assert len(pd.Series(Y_local).unique()) != 1
adaptive_budget = (min(step, budget) * 3) / 4
adaptive_budget += adaptive_budget % 2
random_budget = min(step, budget) - adaptive_budget
predict_func = lambda x: predict(model, x)
samples = utils.line_search(X_local, Y_local,
adaptive_budget / 2, predict_func)
X_random = utils.gen_query_set(X_ext.shape[1], random_budget)
X_ext = numpy.vstack((samples, X_random, X_ext))
if epoch % 1000 == 0:
print("Epoch %d: cost %f" %
(epoch, nn1Layer.cost(X, y, params, decay)))
##======================================================================
## STEP 4: Testing on training data
#
# You should now test your model against the training examples.
# To do this, you will first need to write softmaxPredict
# (in softmax.py), which should return predictions
# given a softmax model and the input data.
pred = nn1Layer.predict(X, params)
acc = np.mean(y == pred)
print('Accuracy: %0.3f%%.' % (acc * 100))
# Accuracy is the proportion of correctly classified images
# After 200 epochs, the results for our implementation were:
#
# Spiral Accuracy: 98.67%
# Flower Accuracy: 86.50%
##======================================================================
## STEP 5: Plot the decision boundary
#
utils.plot_decision_boundary(lambda x: nn1Layer.predict(x, params), X, y)
plt.title("Neural Network")
plt.savefig(FLAGS.input_data + '-boundary.jpg')
plt.show()
#
# Now test your model against the training examples.
# The array pred should contain the predictions of the NN model.
X = X.data.numpy()
W1 = W1.data.numpy()
b1 = b1.data.numpy()
W2 = W2.data.numpy()
b2 = b2.data.numpy()
pred = nn1Layer.predict(X, W1, b1, W2, b2)
acc = np.mean(y == pred)
print('Accuracy: %0.3f%%.' % (acc * 100))
# Accuracy is the proportion of correctly classified images.
# The results for our implementation were:
#
# Spiral Accuracy: 99.00%
# Flower Accuracy: 87.00%
##======================================================================
## STEP 5: Plot the decision boundary
#
utils.plot_decision_boundary(lambda x: nn1Layer.predict(x, W1, b1, W2, b2), X,
y)
plt.title("Neural Network")
plt.savefig(FLAGS.input_data + '-boundary.jpg')
plt.show()
def extract(self, X_train, y_train, budget, steps=[],
adaptive_oracle=False, baseline=False, epsilon=1e-1,
num_passes=1000, reg_lambda=1e-8, eps_factor=0.99, epoch=100,
print_loss=True, print_epoch=10, batch_size=20, random_seed=0):
numpy.random.seed(random_seed)
assert not (adaptive_oracle and steps)
if steps:
step = steps[0]
else:
step = budget
if not adaptive_oracle:
X_ext = utils.gen_query_set(X_train.shape[1], step)
else:
X_ext = utils.line_search_oracle(X_train.shape[1], step, self.query)
idx = 0
while budget > 0:
idx += 1
budget -= step
y_ext = self.query(X_ext)
if not baseline:
y_ext_p = self.query_probas(X_ext)
else:
num_classes = len(self.get_classes())
y_ext_p = numpy.zeros((len(y_ext), num_classes))
y_ext_p[numpy.arange(len(y_ext)), y_ext] = 1
# Loss with correct parameters:
print self.calculate_loss(X_ext, y_ext_p, reg_lambda)
print >> sys.stderr, self.calculate_loss(X_ext, y_ext_p, reg_lambda)
model = build_model(self.hidden_nodes, X_ext, y_ext_p,
epsilon=epsilon, num_passes=num_passes,
reg_lambda=reg_lambda, epoch=epoch,
eps_factor=eps_factor, print_loss=print_loss,
print_epoch=print_epoch, batch_size=batch_size)
m_type = "base" if baseline else "extr"
mode = "adapt-local" if steps \
else "adapt-oracle" if adaptive_oracle \
else "passive"
save(model, 'experiments/{}/models/{}_{}_{}_{}_{}.pkl'.
format(self.dataset, mode, m_type,
self.hidden_nodes, len(X_ext), random_seed))
if X_train is not None and X_train.shape[1] == 2:
bounds = [-1.1, 1.1, -1.1, 1.1]
filename = 'experiments/{}/plots/{}_{}_{}_{}_{}_boundary'.\
format(self.dataset, mode, m_type,
self.hidden_nodes, len(X_ext), random_seed)
utils.plot_decision_boundary(lambda x: predict(model, x),
X_train.values, y_train,
bounds, filename)
filename = 'experiments/{}/plots/{}_{}_{}_{}_{}_boundary_ext'.\
format(self.dataset, mode, m_type,
self.hidden_nodes, len(X_ext), random_seed)
utils.plot_decision_boundary(lambda x: predict(model, x),
X_ext, y_ext, bounds, filename)
if budget > 0:
step = steps[idx] - steps[idx-1]
X_local = utils.gen_query_set(n=X_ext.shape[1], test_size=1000)
Y_local = predict(model, X_local)
assert len(pd.Series(Y_local).unique()) != 1
adaptive_budget = (min(step, budget)*3)/4
adaptive_budget += adaptive_budget % 2
random_budget = min(step, budget) - adaptive_budget
predict_func = lambda x: predict(model, x)
samples = utils.line_search(X_local, Y_local, adaptive_budget/2,
predict_func)
X_random = utils.gen_query_set(X_ext.shape[1], random_budget)
X_ext = numpy.vstack((samples, X_random, X_ext))
def run(self,
data,
X_test,
test_size=100000,
random_seed=0,
alphas=[0.5, 1, 2, 5, 10, 20, 50, 100],
methods=["passive", "adapt-local", "adapt-oracle"],
baseline=True):
np.random.seed(random_seed)
print ','.join(['%s'] * 9) \
% ('dataset', 'method', 'budget', 'mode', 'loss',
'loss_u', 'probas', 'probas_u', 'model l1')
# number of unknown coefficients
k = len(self.get_classes())
n = self.num_features()
num_unknowns = (k - int(k == 2)) * (n + 1)
#num_unknowns = n+1
X_test_u = self.gen_query_set(n, test_size, force_input_space=True)
if k == 2:
for alpha in alphas:
m = int(alpha * num_unknowns)
w_opt, int_opt = self.find_coeffs_bin(budget=m)
# compute the accuracy of the predictions
if X_test is not None:
acc = self.evaluate(w_opt, int_opt, X_test)
l1 = self.evaluate_probas(w_opt, int_opt, X_test)
else:
acc, l1 = [np.nan] * 2
acc_u = self.evaluate(w_opt, int_opt, X_test_u)
l1_u = self.evaluate_probas(w_opt, int_opt, X_test_u)
loss = self.evaluate_model(w_opt, int_opt)
if X_test_u.shape[1] == 2 \
and callable(getattr(self, 'encode', None)):
print X_test_u
utils.plot_decision_boundary(lambda x: predict_classes(
self.encode(x), w_opt, int_opt, self.get_classes()),
X_test_u,
self.query(X_test_u),
bounds=[-1, 1, -1, 1])
print '%s,%s,%d,extr,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, 'binary', m, 1-acc, 1-acc_u,
l1, l1_u, loss)
if baseline and not callable(getattr(self, 'encode', None)):
w_base, int_base, m_base = self.lowd_meek(budget=m)
# compute the accuracy of the predictions
if X_test is not None:
acc_base = self.evaluate(w_base, int_base, X_test)
l1_base = self.evaluate_probas(w_base, int_base,
X_test)
else:
acc_base, l1_base = [np.nan] * 2
acc_u_base = self.evaluate(w_base, int_base, X_test_u)
l1_u_base = self.evaluate_probas(w_base, int_base,
X_test_u)
loss_base = self.evaluate_model(w_base, int_base)
print '%s,%s,%d,base,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, 'lowd-meek', m,
1-acc_base, 1-acc_u_base, l1_base, l1_u_base,
loss_base)
for alpha in alphas:
m = int(alpha * num_unknowns)
step = (m + 4) / 5
for method in methods:
if m < 5:
print '%s,%s,%d,extr,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, method, m, 1.0, 1.0, 1.0, 1.0, 1.0)
print '%s,%s,%d,base,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, method, m, 1.0, 1.0, 1.0, 1.0, 1.0)
continue
base_model = None
if method == "passive":
w_opt, int_opt, m = self.find_coeffs(m)
if baseline:
base_model = self.find_coeffs(m, baseline=True)
elif method == "adapt-local":
w_opt, int_opt = self.find_coeffs_adaptive(step, m)
if baseline:
base_model = self.find_coeffs_adaptive(step,
m,
baseline=True)
elif method == "adapt-oracle":
w_opt, int_opt, m = self.find_coeffs(m, adapt=True)
if baseline:
base_model = self.find_coeffs(m,
baseline=True,
adapt=True)
# compute the accuracy of the predictions
if X_test is not None:
acc = self.evaluate(w_opt,
int_opt,
X_test,
base_model=base_model)
l1 = self.evaluate_probas(w_opt,
int_opt,
X_test,
base_model=base_model)
else:
if baseline:
acc = [np.nan] * 2
l1 = [np.nan] * 2
else:
acc, l1 = np.nan, np.nan
acc_u = self.evaluate(w_opt,
int_opt,
X_test_u,
base_model=base_model)
l1_u = self.evaluate_probas(w_opt,
int_opt,
X_test_u,
base_model=base_model)
loss = self.evaluate_model(w_opt,
int_opt,
base_model=base_model)
if baseline:
print '%s,%s,%d,extr,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, method, m, 1-acc[0], 1-acc_u[0], l1[0],
l1_u[0], loss[0])
print '%s,%s,%d,base,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, method, m, 1-acc[1], 1-acc_u[1], l1[1],
l1_u[1], loss[1])
else:
print '%s,%s,%d,extr,%.2e,%.2e,%.2e,%.2e,%.2e' % \
(data, method, m, 1-acc, 1-acc_u, l1, l1_u, loss)
if X_test_u.shape[1] == 2 \
and callable(getattr(self, 'encode', None)):
utils.plot_decision_boundary(lambda x: predict_classes(
self.encode(x), w_opt, int_opt, self.get_classes()),
X_test_u,
self.query(X_test_u),
bounds=[-1, 1, -1, 1])
def train(self, num_repr, X_test, y_test):
X_train = self.X_train
y_train = self.y_train
y_train_p = np.zeros((len(y_train), len(self.classes)))
y_train_p[np.arange(len(y_train)), y_train] = 1
"""
assert num_repr >= len(self.get_classes())
X_repr_bb = []
class_counter = Counter(y_train)
repr_per_class \
= (num_repr + len(self.get_classes()) - 1) / len(self.get_classes())
for (c, count) in sorted(class_counter.iteritems(), key=lambda _: _[1]):
print c, count, repr_per_class
reprs = X_train.values[np.where(y_train == c)[0][0:repr_per_class]]
X_repr_bb.append(reprs)
X_repr_bb = np.vstack(X_repr_bb)[0:num_repr]
print '{} representers'.format(len(X_repr_bb))
"""
X_repr_bb = X_train[0:num_repr].values
"""
import math
import matplotlib.pyplot as plt
side = math.sqrt(X_repr_bb.shape[1])
plt.figure()
for i in range(len(X_repr_bb)):
plt.subplot(2, len(X_repr_bb)/2, i + 1)
plt.axis('off')
image = X_repr_bb[i, :].reshape((side, side))
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
plt.show()
"""
"""
model_bb = krs.build_model(X_repr_bb, False, X_train, y_train_p,
epsilon=epsilon, reg_lambda=1e-4,
num_passes=num_passes, eps_factor=0.99,
epoch=100, print_epoch=10, batch_size=1,
gamma=gamma)
y_pred_bb = krs.predict(model_bb, X_test)
"""
best_model = None
best_acc = 0
best_gamma = None
for gamma in [10**x for x in range(-10, 10)]:
from sklearn.metrics.pairwise import rbf_kernel
from sklearn.linear_model import LogisticRegression
X_train_ker = rbf_kernel(X_train, X_repr_bb, gamma=gamma)
model = LogisticRegression(multi_class='multinomial',
solver='lbfgs').\
fit(X_train_ker, y_train)
y_pred = model.predict(rbf_kernel(X_test, X_repr_bb, gamma=gamma))
acc = accuracy_score(y_test, y_pred)
if acc > best_acc:
best_acc = acc
best_gamma = gamma
best_model = model
W = best_model.coef_.T
b = best_model.intercept_
if len(self.classes) == 2:
W = np.hstack((np.zeros((len(X_repr_bb), 1)), W))
b = np.hstack((0, b))
W = theano.shared(
value=W,
name='W',
borrow=True
)
b = theano.shared(
value=b,
name='b',
borrow=True
)
model_bb = krs.KernelLog(T.matrix('x'), best_gamma, len(self.classes),
X_repr_bb, learn_Y=False, W=W, b=b)
y_pred_bb = krs.predict(model_bb, X_test)
print 'Best Gamma: {}'.format(best_gamma)
print 'Y_test: {}'.format(Counter(y_test))
print 'Y_pred: {}'.format(Counter(y_pred_bb))
acc = accuracy_score(y_test, y_pred_bb)
print 'Training accuracy: {}'.format(acc)
print >> sys.stderr, 'Training accuracy: {}'.format(acc)
X_test_u = utils.gen_query_set(X_test.shape[1], 10000)
y_pred_u = krs.predict(model_bb, X_test_u)
print 'Y_pred_u: {}'.format(Counter(y_pred_u))
if X_train.shape[1] == 2:
bounds = [-1.1, 1.1, -1.1, 1.1]
X_train = X_train.values
utils.plot_decision_boundary(
lambda x: krs.predict(model_bb, x), X_train, y_train, bounds)
krs.save(model_bb,
'experiments/KLR/{}/models/oracle.pkl'.format(self.dataset))
def run():
#size of minibatch: int(X.shape[0])
N = 50
X, y = load_XOR_data(N=300)
input_dim = int(X.shape[1])
hidden_dim = 10
output_dim = 2
num_epochs = 1000000
learning_rate = 1e-3
reg_coeff = 1e-6
losses = []
accuracies = []
#---------------------------------------------------------------------------------------------------------------
# Initialize weights:
np.random.seed(2017)
w1 = 2.0 * np.random.random(
(input_dim, hidden_dim)) - 1.0 #w0=(2,hidden_dim)
w2 = 2.0 * np.random.random(
(hidden_dim, output_dim)) - 1.0 #w1=(hidden_dim,2)
#Calibratring variances with 1/sqrt(fan_in)
w1 /= np.sqrt(input_dim)
w2 /= np.sqrt(hidden_dim)
for i in range(num_epochs):
index = np.arange(X.shape[0])[:N]
#is want to shuffle indices: np.random.shuffle(index)
#---------------------------------------------------------------------------------------------------------------
# Forward step:
h1 = sigmoid(np.matmul(X[index], w1)) #(N, 3)
logits = sigmoid(np.matmul(h1, w2)) #(N, 2)
probs = np.exp(logits) / np.sum(np.exp(logits), axis=1, keepdims=True)
h2 = logits
#---------------------------------------------------------------------------------------------------------------
# Definition of Loss function: mean squared error plus Ridge regularization
L = np.square(y[index] - h2).sum() / (2 * N) + reg_coeff * (
np.square(w1).sum() + np.square(w2).sum()) / (2 * N)
losses.append([i, L])
#---------------------------------------------------------------------------------------------------------------
# Backward step: Error = W_l e_l+1 f'_l
# dL/dw2 = dL/dh2 * dh2/dz2 * dz2/dw2
dL_dh2 = -(y[index] - h2) #(N, 2)
dh2_dz2 = sigmoid(h2, first_derivative=True) #(N, 2)
dz2_dw2 = h1 #(N, hidden_dim)
#Gradient for weight2: (hidden_dim,N)x(N,2)*(N,2)
dL_dw2 = dz2_dw2.T.dot(
dL_dh2 * dh2_dz2) + reg_coeff * np.square(w2).sum()
#dL/dw1 = dL/dh1 * dh1/dz1 * dz1/dw1
# dL/dh1 = dL/dz2 * dz2/dh1
# dL/dz2 = dL/dh2 * dh2/dz2
dL_dz2 = dL_dh2 * dh2_dz2 #(N, 2)
dz2_dh1 = w2 #z2 = h1*w2
dL_dh1 = dL_dz2.dot(dz2_dh1.T) #(N,2)x(2, hidden_dim)=(N, hidden_dim)
dh1_dz1 = sigmoid(h1, first_derivative=True) #(N,hidden_dim)
dz1_dw1 = X[index] #(N,2)
#Gradient for weight1: (2,N)x((N,hidden_dim)*(N,hidden_dim))
dL_dw1 = dz1_dw1.T.dot(
dL_dh1 * dh1_dz1) + reg_coeff * np.square(w1).sum()
#weight updates:
w2 += -learning_rate * dL_dw2
w1 += -learning_rate * dL_dw1
if True: #(i+1)%1000==0:
y_pred = inference(X, [w1, w2])
y_actual = np.argmax(y, axis=1)
accuracy = np.sum(np.equal(y_pred, y_actual)) / len(y_actual)
accuracies.append([i, accuracy])
if (i + 1) % 10000 == 0:
print('Epoch %d\tLoss: %f Average L1 error: %f Accuracy: %f' %
(i, L, np.mean(np.abs(dL_dh2)), accuracy))
save_filepath = './scratch_mlp/plots/boundary/image_%d.png' % i
text = 'Batch #: %d Accuracy: %.2f Loss value: %.2f' % (
i, accuracy, L)
utils.plot_decision_boundary(X,
y_actual,
lambda x: inference(x, [w1, w2]),
save_filepath=save_filepath,
text=text)
save_filepath = './scratch_mlp/plots/loss/image_%d.png' % i
utils.plot_function(losses,
save_filepath=save_filepath,
ylabel='Loss',
title='Loss estimation')
save_filepath = './scratch_mlp/plots/accuracy/image_%d.png' % i
utils.plot_function(accuracies,
save_filepath=save_filepath,
ylabel='Accuracy',
title='Accuracy estimation')
def extract(self,
X_train,
y_train,
num_repr,
budget,
steps=[],
adaptive_oracle=False,
use_labels_only=False,
gamma=1,
epsilon=1e-2,
reg_lambda=1e-16,
eps_factor=0.99,
epoch=100,
print_epoch=10,
batch_size=1,
num_passes=1000,
random_seed=0):
"""
Extracts a logistic regression multilabl classifier with RBF kernel
for self.query
:param X_train: For plotting
:param y_train: For plotting
:param num_repr: amount of vectors in the kernel base.
:param budget: total amount of queries to the oracle.
:param steps: monotoincally increasing list of queries amounts.
:param adaptive_oracle: whether to query the oracle using line search,
or use only random inputs.
:param use_labels_only: Whether to use only the labels in training.
:param gamma: coefficient for the RBF kernel
:param epsilon: initial learining rate.
:param reg_lambda: regularization coefficient.
:param eps_factor: update factor for the learning rate.
Once in every epoch:
learning_rate *= eps_factor
:param epoch: how many iterations over the whole training data count as a n epoch.
:param print_epoch:
:param batch_size:
:param num_passes: how many iterations over the whole data should we do in each training.
:param random_seed:
:return:
"""
numpy.random.seed(random_seed)
assert not (adaptive_oracle and steps)
if steps:
step = steps[0]
else:
step = budget
if not adaptive_oracle:
X_ext = utils.gen_query_set(self.num_features(), step)
else:
X_ext = utils.line_search_oracle(self.num_features(), step,
self.query)
X_repr = utils.gen_query_set(self.num_features(), num_repr)
model = None
idx = 0
while budget > 0:
idx += 1
budget -= step
y_ext = self.query(X_ext)
if not use_labels_only:
y_ext_p = self.query_probas(X_ext)
else:
# When this a baseline, take the labels from the query result
# as one-hot vectors to be the probability vectors.
num_classes = len(self.get_classes())
y_ext_p = numpy.zeros((len(y_ext), num_classes))
y_ext_p[numpy.arange(len(y_ext)), y_ext] = 1
print y_ext_p
print '{} ({})'.format(
self.calculate_loss(X_ext, y_ext_p, reg_lambda),
self.calculate_loss(X_ext, y_ext_p, 0))
print >> sys.stderr, self.calculate_loss(X_ext, y_ext_p,
reg_lambda)
model = build_model(X_repr,
False,
X_ext,
y_ext_p,
gamma=gamma,
epsilon=epsilon,
reg_lambda=reg_lambda,
num_passes=num_passes,
eps_factor=eps_factor,
epoch=epoch,
print_epoch=print_epoch,
batch_size=batch_size,
warm_start=model)
mtype = "base" if use_labels_only else "extr"
mode = "adapt-local" if steps \
else "adapt-oracle" if adaptive_oracle \
else "passive"
save(
model, 'experiments/KLR/{}/models/{}_{}_{}.pkl'.format(
self.dataset, mode, mtype, len(X_ext)))
if X_train is not None and X_train.shape[1] == 2:
bounds = [-1.1, 1.1, -1.1, 1.1]
filename = 'experiments/KLR/{}/plots/{}_{}_{}_{}_boundary'.\
format(self.dataset, mode, mtype, len(X_ext), random_seed)
utils.plot_decision_boundary(lambda x: predict(model, x),
X_train.values, y_train, bounds,
filename)
filename = 'experiments/KLR/{}/plots/{}_{}_{}_{}_boundary_ext'.\
format(self.dataset, mode, mtype, len(X_ext), random_seed)
utils.plot_decision_boundary(lambda x: predict(model, x),
X_ext, y_ext, bounds, filename)
if budget > 0:
step = steps[idx] - steps[idx - 1]
X_local = utils.gen_query_set(n=X_repr.shape[1],
test_size=1000)
Y_local = predict(model, X_local)
assert len(pd.Series(Y_local).unique()) != 1
adaptive_budget = (min(step, budget) * 3) / 4
adaptive_budget += adaptive_budget % 2
random_budget = min(step, budget) - adaptive_budget
predict_func = lambda x: predict(model, x)
samples = utils.line_search(X_local, Y_local,
adaptive_budget / 2, predict_func)
X_random = utils.gen_query_set(X_ext.shape[1], random_budget)
X_ext = numpy.vstack((samples, X_random, X_ext))
cache, al = self.forward_propagation(self.parameters, X)
predicts = (al > 0.5)
return predicts
def score(self, X, Y):
predicts = self.predict(X)
accuracy = float(
(np.dot(Y, predicts.T) + np.dot(1 - Y, 1 - predicts.T)) /
float(Y.size))
return accuracy
# test
if __name__ == '__main__':
from utils import load_planar_dataset, plot_decision_boundary
np.random.seed(1)
X, Y = load_planar_dataset()
nn = DNN(X=X,
Y=Y,
layer_dims=[2, 4, 4, 1],
max_iter=10000,
alpha=1.2,
print_loss=True,
activation='tanh')
nn.fit()
predicts = nn.predict(X)
accuracy = float((np.dot(Y, predicts.T) + np.dot(1 - Y, 1 - predicts.T)) /
float(Y.size) * 100)
print('Accuracy: %f ' % accuracy + '%')
plot_decision_boundary(lambda x: nn.predict(x.T), X, Y,
'tanh[2881]=' + str(accuracy) + '%')
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import sklearn.datasets
import sklearn.linear_model
import mlnn
from utils import plot_decision_boundary
# Generate a dataset and plot it
np.random.seed(0)
X, y = sklearn.datasets.make_moons(200, noise=0.20)
plt.scatter(X[:,0], X[:,1], s=40, c=y, cmap=plt.cm.Spectral)
plt.show()
layers_dim = [2, 3, 2]
model = mlnn.Model(layers_dim)
model.train(X, y, num_passes=20000, epsilon=0.01, reg_lambda=0.01, print_loss=True)
# Plot the decision boundary
plot_decision_boundary(lambda x: model.predict(x), X, y)
plt.title("Decision Boundary for hidden layer size 3")
plt.show()
X, y = make_moons(200, noise=0.2, random_state=0)
# Plot data
ax = plt.gca()
utils.plot_data(ax, X, y, title='Original Moons Data')
plt.show()
#-------------------------------------------------------------------------------
# Part 4 - Testing our Neural Network
#-------------------------------------------------------------------------------
nn = lab8.NN(n_input=X.shape[1], n_hidden=50, n_output=2)
nn.train(X, y, num_epochs=100, verbose=True)
ax = plt.gca()
utils.plot_decision_boundary(ax, X, y, nn, title='Neural Network Moons')
plt.show()
#-------------------------------------------------------------------------------
# Part 5 - Experimenting with the Number of Neurons
#-------------------------------------------------------------------------------
hidden_layer_dimensions = [1, 2, 3, 4, 5, 6, 10, 20, 50]
for i, n_hidden in tqdm(enumerate(hidden_layer_dimensions),
total=len(hidden_layer_dimensions)):
nn = lab8.NN(n_input=X.shape[1], n_hidden=n_hidden, n_output=2)
nn.train(X, y, num_epochs=2000)
ax = plt.subplot(3, 3, i + 1)
utils.plot_decision_boundary(
ax,
# Display the original data
plt.scatter(x1[:,0], x1[:,1], s=10., label='N1')
plt.scatter(x2[:,0], x2[:,1], s=10., label='N2')
plt.legend()
# Now, we duplicate in two the dataset to use in each procedure
# random sampling with data_rs, and uncertainty sampling with data_us
data_ms = np.copy(x)
#################### ACTIVE LEARNING ####################
evaluation_ms, weights_ms, training_ms = active_learning(data_ms, n_iter=10, n_sample=10, epochs=500, acquisition_function=sample_lowest_margin)
plt.plot(evaluation_ms)
line_ms_x, line_ms_y = plot_decision_boundary(weights_ms)
# Plot the decision boundary learned with the uniformly sampled data
id0 = training_ms[:,2] == 0
id1 = training_ms[:,2] == 1
rs_id0 = training_ms[id0]
rs_id1 = training_ms[id1]
# Start with just the data we trained with
plt.scatter(rs_id0[:,0], rs_id0[:,1], s=30., label='N1')
plt.scatter(rs_id1[:,0], rs_id1[:,1], s=30., label='N2')
plt.plot(line_ms_x, line_ms_y, label="Margin Sampling")
plt.xlim(-5,5)
plt.ylim(-4,4)
plt.title("Decision Boundary with the data we trained on ")
plt.legend()
import numpy as np
import sklearn
import sklearn.datasets
import sklearn.linear_model
import mlnn
from utils import plot_decision_boundary
from nn_log import nn_log_instance
# Generate a dataset and plot it
np.random.seed(0)
X, y = sklearn.datasets.make_moons(200, noise=0.2, shuffle=False)
plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)
plt.show()
layers_dim = [2, 3, 2]
model = mlnn.Model(layers_dim)
model.train(X,
y,
num_passes=20000,
epsilon=0.01,
reg_lambda=0.01,
print_loss=True)
# Plot the decision boundary
plot_decision_boundary(lambda x: model.predict(x), X, y)
plt.title("Decision Boundary for hidden layer size 3")
plt.show()
nn_log_instance.close_file()
############################################################################
svm = LinearSVM_twoclass()
svm.theta = np.zeros((XX.shape[1],))
svm.train(XX,yy,learning_rate=1e-4,C=C,num_iters=50000,verbose=True)
# classify the training data
y_pred = svm.predict(XX)
print "Accuracy on training data = ", metrics.accuracy_score(yy,y_pred)
# visualize the decision boundary
utils.plot_decision_boundary(scaleX,y,svm,'x1','x2',['neg','pos'])
plt.savefig('fig2.pdf')
############################################################################
# Part 3: Training SVM with a kernel #
# We train an SVM with an RBF kernel on the data set and the plot the #
# learned decision boundary #
############################################################################
# test your Gaussian kernel implementation
x1 = np.array([1,2,1])
x2 = np.array([0,4,-1])
sigma = 2
print "Guassian kernel value (should be around 0.324652) = ", utils.gaussian_kernel(x1,x2,sigma)
############################################################################
svm = LinearSVM_twoclass()
svm.theta = np.zeros((XX.shape[1], ))
svm.train(XX, yy, learning_rate=1e-4, C=C, num_iters=50000, verbose=True)
# classify the training data
y_pred = svm.predict(XX)
print "Accuracy on training data = ", metrics.accuracy_score(yy, y_pred)
# visualize the decision boundary
utils.plot_decision_boundary(scaleX, y, svm, 'x1', 'x2', ['neg', 'pos'])
plt.savefig('fig2.pdf')
############################################################################
# Part 3: Training SVM with a kernel #
# We train an SVM with an RBF kernel on the data set and the plot the #
# learned decision boundary #
############################################################################
# test your Gaussian kernel implementation
x1 = np.array([1, 2, 1])
x2 = np.array([0, 4, -1])
sigma = 2
print "Guassian kernel value (should be around 0.324652) = ", utils.gaussian_kernel(
def exe_3_2_1():
"""Experiment varying the number of hidden layers."""
# Params.
epochs = 500
n = 110
mA, mB = [2.0, 0.5], [-2.0, -1.5]
sigmaA, sigmaB = 0.5, 0.5
output_layer_size = 1
# Experiment 1 - Changing number of hidden nodes
train_accuracies, train_losses = [], []
# Generate data.
data = generate_linear_data(n,
mA,
mB,
sigmaA,
sigmaB,
target_values=[1, -1])
inputs, targets = data['inputs'], data['targets']
x_train, x_val, y_train, y_val = train_test_split_class(
inputs, targets, split=0.2, split_valance=[0.5, 0.5])
# Create and train network with different number of hidden layers.
for hidden_layer_shape in range(1, 30):
model = NueralNet(x_train,
y_train,
hidden_layer_size=hidden_layer_shape,
output_layer_size=output_layer_size)
losses = model.train_network(epochs, x_val, y_val)
train_accuracies.append(losses["epoch_accuracies"][-1])
train_losses.append(losses["epoch_losses"][-1])
# Plot results.
plt.plot(train_losses, label="Train losses")
plt.xlabel("Number of hidden nodes.")
plt.ylabel("Mean Squared Error loss")
plt.show()
plt.plot(train_accuracies, label="Train Accuracies")
plt.xlabel("Number of hidden nodes.")
plt.ylabel("Mean Squared Error loss")
plt.show()
# Experiment 2 - Changing train test split ratio
split_ratios = [0.2, 0.4, 0.6, 0.8]
hidden_layer_shape = 8
for split in split_ratios:
# Generate data.
# data = generate_linear_data(n, mA, mB, sigmaA, sigmaB, target_values = [1, -1])
data = generate_nonlinear_data(n,
mA,
mB,
sigmaA,
sigmaB,
target_values=[1, -1])
inputs, targets = data['inputs'], data['targets']
# x_train, x_val, y_train, y_val = train_test_split(inputs, targets, split = split)
x_train, x_val, y_train, y_val = train_test_split_class(
inputs, targets, split=split, split_valance=[0.5, 0.5])
model = NueralNet(x_train,
y_train,
hidden_layer_size=hidden_layer_shape,
output_layer_size=output_layer_size)
losses = model.train_network(epochs, x_val, y_val)
plot_losses(losses, f"Data split: {split}")
# Experiment 3 - Change both train test split ratio and number of hidden nodes.
# Experiment 4 - Check difference between sequential and batch learning.
# Experiment 5 - Plot decision boundary.
hidden_layer_shape = 18
output_layer_size = 1
epochs = 500
data = generate_nonlinear_data(n,
mA,
mB,
sigmaA,
sigmaB,
target_values=[1, -1])
inputs, targets = data["inputs"], data["targets"]
x_train, x_val, y_train, y_val = train_test_split(inputs,
targets,
split=0.2)
model = NueralNet(
x_train,
y_train,
hidden_layer_size=hidden_layer_shape,
output_layer_size=output_layer_size,
)
model.train_network(epochs)
plot_decision_boundary(inputs, targets, model)
