def CAL_v(name, label_p, label_n, oracle, n_features, ftype, test_x, test_y):
online = OnlineBase(name, label_p, label_n, oracle, n_features, ftype, error=.5)
x, y = online.collect_pts(100, -1)
i = 0
q = online.get_n_query()
C_range = np.logspace(-2, 5, 10, base=10)
gamma_range = np.logspace(-5, 1, 10, base=10)
param_grid = dict(gamma=gamma_range, C=C_range)
while q < 3500:
i += 1
# h_ = ex.fit(x, y)
cv = StratifiedShuffleSplit(y, n_iter=5, test_size=0.2, random_state=42)
grid = GridSearchCV(svm.SVC(), param_grid=param_grid, cv=cv, verbose=0, n_jobs=-1)
grid.fit(x, y)
h_ = grid.best_estimator_
online_ = OnlineBase('', label_p, label_n, h_.predict, n_features, ftype, error=.1)
x_, _ = online_.collect_pts(10, 200)
if x_ is not None and len(x_) > 0:
x.extend(x_)
y.extend(oracle(x_))
q += online_.get_n_query()
pred_y = h_.predict(test_x)
print len(x), q, sm.accuracy_score(test_y, pred_y)
def retrain_in_f_with_grid(name, label_p, label_n, oracle, n_features, ftype,
test_x, test_y, benchmark):
print '--------------- retrain in F with grid -----------------'
for n_pts in xrange(50, 601, 50):
online = OnlineBase(name,
label_p,
label_n,
oracle,
n_features,
ftype,
error=.1)
online.collect_pts(n_pts, -1)
ex = RBFKernelRetraining(
name,
online.get_QSV(),
online.get_QSV_labels(), # training data
online.get_QSV(),
online.get_QSV_labels(), # validation data
test_x,
test_y, # test data
n_features)
print 'nQSV=%d, Q=%d, dim=100,' % (
n_pts, online.get_n_query()), ex.grid_retrain_in_f(100)
def CAL_v(name, label_p, label_n, oracle, n_features, ftype, test_x, test_y):
online = OnlineBase(name,
label_p,
label_n,
oracle,
n_features,
ftype,
error=.5)
x, y = online.collect_pts(100, -1)
i = 0
q = online.get_n_query()
C_range = np.logspace(-2, 5, 10, base=10)
gamma_range = np.logspace(-5, 1, 10, base=10)
param_grid = dict(gamma=gamma_range, C=C_range)
while q < 3500:
i += 1
# h_ = ex.fit(x, y)
cv = StratifiedShuffleSplit(y,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h_ = grid.best_estimator_
online_ = OnlineBase('',
label_p,
label_n,
h_.predict,
n_features,
ftype,
error=.1)
x_, _ = online_.collect_pts(10, 200)
if x_ is not None and len(x_) > 0:
x.extend(x_)
y.extend(oracle(x_))
q += online_.get_n_query()
pred_y = h_.predict(test_x)
print len(x), q, sm.accuracy_score(test_y, pred_y)
def CAL(name, label_p, label_n, oracle, n_features, ftype, test_x, test_y):
online = OnlineBase(name, label_p, label_n, oracle, n_features, ftype, error=.5)
q = online.get_n_query()
C_range = np.logspace(-2, 5, 10, base=10)
gamma_range = np.logspace(-5, 1, 10, base=10)
param_grid = dict(gamma=gamma_range, C=C_range)
x, y = online.collect_pts(100, -1)
i = 0
cv = StratifiedShuffleSplit(y, n_iter=5, test_size=0.2, random_state=42)
grid = GridSearchCV(svm.SVC(), param_grid=param_grid, cv=cv, verbose=0, n_jobs=-1)
grid.fit(x, y)
h_ = grid.best_estimator_
while q < 3500:
i += 1
# h_ = ex.fit(x, y)
online_ = OnlineBase('', label_p, label_n, h_.predict, n_features, ftype, error=.1)
x_ = online_.collect_one_pair()
if x_ is not None and len(x_) > 0:
for _x in x_:
x.append(_x)
y.append(1)
cv = StratifiedShuffleSplit(y, n_iter=5, test_size=0.2, random_state=42)
grid = GridSearchCV(svm.SVC(), param_grid=param_grid, cv=cv, verbose=0, n_jobs=-1)
grid.fit(x, y)
h1 = grid.best_estimator_
s1 = sm.accuracy_score(y, h1.predict(x))
y[-1] = -1
cv = StratifiedShuffleSplit(y, n_iter=5, test_size=0.2, random_state=42)
grid = GridSearchCV(svm.SVC(), param_grid=param_grid, cv=cv, verbose=0, n_jobs=-1)
grid.fit(x, y)
h2 = grid.best_estimator_
s2 = sm.accuracy_score(y, h2.predict(x))
if s1 >= .99 and s2 >= .99:
print 'branch 1'
y[-1] = oracle(x_)[0]
elif s1 >= .99 and s2 < .99:
print 'branch 2'
y[-1] = 1
elif s1 < .99 and s2 >= .99:
print 'branch 3'
y[-1] = -1
else:
print 'branch 4: ', s1, s2
del x[-1]
del y[-1]
continue
if y[-1] == 1:
h_ = h1
else:
h_ = h2
q += online_.get_n_query()
pred_y = h_.predict(test_x)
print q, sm.accuracy_score(test_y, pred_y)
def CAL(name, label_p, label_n, oracle, n_features, ftype, test_x, test_y):
"""
Learn with adaptive learning the oracle, using an SVM
with RBF kernel,
prints the accuracy as function of amount of queries to
the LOCAL MODEL (weird function).
:param name:
:param label_p:
:param label_n:
:param oracle:
:param n_features:
:param ftype:
:param test_x:
:param test_y:
:return:
"""
online = OnlineBase(name,
label_p,
label_n,
oracle,
n_features,
ftype,
error=.5)
# This is weird - the count should be zero here.
q = online.get_n_query()
C_range = np.logspace(-2, 5, 10, base=10)
gamma_range = np.logspace(-5, 1, 10, base=10)
param_grid = dict(gamma=gamma_range, C=C_range)
x, y = online.collect_pts(100, -1)
i = 0
cv = StratifiedShuffleSplit(y, n_iter=5, test_size=0.2, random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h_ = grid.best_estimator_
while q < 3500:
i += 1
# h_ = ex.fit(x, y)
# This is not really an online model - we set oracle=h_.predict.
local_model = OnlineBase('',
label_p,
label_n,
h_.predict,
n_features,
ftype,
error=.1)
x_ = local_model.collect_one_pair()
if x_ is not None and len(x_) > 0:
for _x in x_:
#
x.append(_x)
y.append(1)
cv = StratifiedShuffleSplit(y,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h1 = grid.best_estimator_
s1 = sm.accuracy_score(y, h1.predict(x))
y[-1] = -1
cv = StratifiedShuffleSplit(y,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h2 = grid.best_estimator_
s2 = sm.accuracy_score(y, h2.predict(x))
# Assume implicitly that the local model can reach
# over 99% accuracy over the training set.
# Check whether there is a reason the query the oracle about x_:
# * If for a specific prediction, the performance of
# of the model over the so-far found points will
# degrade under 99%, it would be useless to query the
# oracle because we can already guess this prediction
# is wrong.
# * Otherwise, we are not certain about oracle(x_) - so we
# query the oracle.
# Very weird - add the point as training point anyway,
# also when we guess oracle(x_).
# Notice: I expect that most of the times, only
# the first "if" will take effect and actually run,
# Because the points are really close to each other.
if s1 >= .99 and s2 >= .99:
print 'branch 1'
y[-1] = oracle(x_)[0]
elif s1 >= .99 and s2 < .99:
print 'branch 2'
y[-1] = 1
elif s1 < .99 and s2 >= .99:
print 'branch 3'
y[-1] = -1
else:
print 'branch 4: ', s1, s2
del x[-1]
del y[-1]
continue
if y[-1] == 1:
h_ = h1
else:
h_ = h2
# This is weird - why do we count the queries of the local_model ?
# I think we should count the queries to the oracle !
q += local_model.get_n_query()
pred_y = h_.predict(test_x)
print q, sm.accuracy_score(test_y, pred_y)
def CAL_v(name, label_p, label_n, oracle, n_features, ftype, test_x, test_y):
"""
Prints the test accuracy of an RBF-kernel SVM predictor
for a varying amount of "points near the boundary"
[boundary of the oracle].
:param name:
:param label_p:
:param label_n:
:param oracle:
:param n_features:
:param ftype:
:param test_x:
:param test_y:
:return:
"""
online = OnlineBase(name,
label_p,
label_n,
oracle,
n_features,
ftype,
error=.5)
x, y = online.collect_pts(100, -1)
i = 0
q = online.get_n_query()
C_range = np.logspace(-2, 5, 10, base=10)
gamma_range = np.logspace(-5, 1, 10, base=10)
param_grid = dict(gamma=gamma_range, C=C_range)
while q < 3500:
i += 1
# h_ = ex.fit(x, y)
cv = StratifiedShuffleSplit(y,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h_ = grid.best_estimator_
online_ = OnlineBase('',
label_p,
label_n,
h_.predict,
n_features,
ftype,
error=.1)
x_, _ = online_.collect_pts(10, 200)
if x_ is not None and len(x_) > 0:
x.extend(x_)
y.extend(oracle(x_))
q += online_.get_n_query()
pred_y = h_.predict(test_x)
print "total amount of ", len(x), q, sm.accuracy_score(test_y, pred_y)
def main():
X1, Y1 = make_circles(n_samples=800, noise=0.07,
factor=0.4) # defined in sklearn.datasets
# gererates a data set X1 and labels Y1 with data from two circles, an inner circle
# and an outer circle. The labels in Y1 are 0 or 1, indiciating the inner or outer circle.
# n_samples is the number of data points, noise is the noise on the data, factor is the
# ratio between the radius of the inner circle to the radius of the outer circle
frac0 = len(np.where(Y1 == 0)[0]) / float(
len(Y1)) # the number of points in the inner circle
frac1 = len(np.where(Y1 == 1)[0]) / float(
len(Y1)) # the number of points in the outer circle
print("Percentage of '0' labels:", frac0)
print("Percentage of '1' labels:", frac1)
plt.figure()
plt.subplot(121)
plt.title(
"Our Dataset: N=200, '0': {0} '1': {1} ".format(
frac0, frac1), # format is a way of printing reals/integers
fontsize="large")
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)
plt.xlim((-2, 2))
plt.ylim((-2, 2))
clf = svm.SVC() # creates a support vector classification object.
clf.fit(X1, Y1) # fits the SVC to the data given
print(accuracy_score(Y1, clf.predict(
X1))) # prints the accuracy of the model on the training data
ex = OnlineBase('circle', 1, 0, clf.predict, 2, 'uniform', .1)
step = 6
train_x, train_y = [], []
val_x, val_y = [], []
while True:
ex.collect_pts(
step) # collects step points around the decision boundary of ex
train_x.extend(ex.pts_near_b) # first step this list is empty.
train_y.extend(ex.pts_near_b_labels) # first step this list is empty
#val_x.extend(ex.support_pts)
#val_y.extend(ex.support_labels)
try:
e = RBFKernelRetraining(
'circle', [train_x, train_y], [train_x, train_y], n_features=2
) # creates a new object every time? is this the smartest way to retrain?
print(
ex.get_n_query(), e.grid_retrain_in_x()
) # TODO I do not get how ex and e are connected, it seems to me that
# grid_retrain_in_x() indeeds does something like retraing the model, but there are no points added to pts_near_b or are there?
except KeyboardInterrupt: ## TODO stop condition!!
print('Done')
break
train_x = np.array(train_x)
plt.subplot(122)
plt.scatter(train_x[:, 0], train_x[:, 1], c=train_y)
plt.xlim((-2, 2))
plt.ylim((-2, 2))
plt.show()
def CAL(name, label_p, label_n, oracle, n_features, ftype, test_x, test_y):
online = OnlineBase(name,
label_p,
label_n,
oracle,
n_features,
ftype,
error=.5)
q = online.get_n_query()
C_range = np.logspace(-2, 5, 10, base=10)
gamma_range = np.logspace(-5, 1, 10, base=10)
param_grid = dict(gamma=gamma_range, C=C_range)
x, y = online.collect_pts(100, -1)
i = 0
cv = StratifiedShuffleSplit(y, n_iter=5, test_size=0.2, random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h_ = grid.best_estimator_
while q < 3500:
i += 1
# h_ = ex.fit(x, y)
online_ = OnlineBase('',
label_p,
label_n,
h_.predict,
n_features,
ftype,
error=.1)
x_ = online_.collect_one_pair()
if x_ is not None and len(x_) > 0:
for _x in x_:
x.append(_x)
y.append(1)
cv = StratifiedShuffleSplit(y,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h1 = grid.best_estimator_
s1 = sm.accuracy_score(y, h1.predict(x))
y[-1] = -1
cv = StratifiedShuffleSplit(y,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(svm.SVC(),
param_grid=param_grid,
cv=cv,
verbose=0,
n_jobs=-1)
grid.fit(x, y)
h2 = grid.best_estimator_
s2 = sm.accuracy_score(y, h2.predict(x))
if s1 >= .99 and s2 >= .99:
print 'branch 1'
y[-1] = oracle(x_)[0]
elif s1 >= .99 and s2 < .99:
print 'branch 2'
y[-1] = 1
elif s1 < .99 and s2 >= .99:
print 'branch 3'
y[-1] = -1
else:
print 'branch 4: ', s1, s2
del x[-1]
del y[-1]
continue
if y[-1] == 1:
h_ = h1
else:
h_ = h2
q += online_.get_n_query()
pred_y = h_.predict(test_x)
print q, sm.accuracy_score(test_y, pred_y)