def test_cross_validation_without_self_partitioning_ok(self):
split_count = 5
instance_num = 100
self.__X, self.__y = make_classification(n_samples=instance_num,
n_features=4,
n_informative=2,
n_redundant=2,
n_repeated=0,
n_classes=2,
n_clusters_per_class=2,
weights=None,
flip_y=0.01,
class_sep=1.0,
hypercube=True,
shift=0.0,
scale=1.0,
shuffle=True,
random_state=None)
train_idx, test_idx, label_idx, unlabel_idx = split(
X=self.__X,
y=self.__y,
test_ratio=0.3,
initial_label_rate=0.05,
split_count=split_count,
all_class=True)
# init the AlExperiment
experiment = CrossValidationExperiment(
self.__X,
self.__y,
self_partition=False,
stopping_criteria=UnlabelSetEmpty(),
train_idx=train_idx,
test_idx=test_idx,
label_idx=label_idx,
unlabel_idx=unlabel_idx)
assert len(experiment._train_idx) == split_count
assert len(experiment._test_idx) == split_count
assert len(experiment._label_idx) == split_count
assert len(experiment._unlabel_idx) == split_count
for i in range(split_count):
train = set(experiment._train_idx[i])
test = set(experiment._test_idx[i])
lab = set(experiment._label_idx[i])
unl = set(experiment._unlabel_idx[i])
assert len(test) == round(0.3 * instance_num)
assert len(lab) == round(0.05 * len(train))
# validity
traintest = train.union(test)
labun = lab.union(unl)
assert traintest == set(range(instance_num))
assert labun == train
def test_query_regression_std_batch_size(self):
# Get the data
X = np.random.choice(np.linspace(0, 20, 1000), size=100, replace=False).reshape(-1, 1)
y = np.sin(X) + np.random.normal(scale=0.3, size=X.shape)
# assembling initial training set
train_idx, test_idx, label_idx, unlabel_idx = split(
X=X,
y=y,
test_ratio=0.3,
initial_label_rate=0.05,
split_count=1,
all_class=True)
# defining the kernel for the Gaussian process
ml_technique = GaussianProcessRegressor(
kernel=RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \
+ WhiteKernel(noise_level=1, noise_level_bounds=(1e-10, 1e+1)))
experiment = HoldOutExperiment(
client=self.__client,
X=X,
Y=y,
scenario_type=PoolBasedSamplingScenario,
train_idx=train_idx,
test_idx=test_idx,
label_idx=label_idx,
unlabel_idx=unlabel_idx,
ml_technique=ml_technique,
performance_metrics=[Mse(squared=True)],
query_strategy=QueryRegressionStd(),
oracle=SimulatedOracle(labels=y),
stopping_criteria=PercentOfUnlabel(value=70),
self_partition=False,
batch_size=self.__batch_size
)
result = experiment.evaluate(verbose=True)
regressor = result[0].ml_technique
# plotting the initial estimation
with plt.style.context('seaborn-white'):
plt.figure(figsize=(14, 7))
x = np.linspace(0, 20, 1000)
pred, std = regressor.predict(x.reshape(-1, 1), return_std=True)
plt.plot(x, pred)
plt.fill_between(x, pred.reshape(-1, ) - std, pred.reshape(-1, ) + std, alpha=0.2)
plt.scatter(X, y, c='k')
plt.title('Initial estimation')
plt.show()
def test_cross_validation_without_self_partitioning_wrong_kfold_size(self):
split_count = 5
instance_num = 100
self.__X, self.__y = make_classification(n_samples=instance_num,
n_features=4,
n_informative=2,
n_redundant=2,
n_repeated=0,
n_classes=2,
n_clusters_per_class=2,
weights=None,
flip_y=0.01,
class_sep=1.0,
hypercube=True,
shift=0.0,
scale=1.0,
shuffle=True,
random_state=None)
train_idx, test_idx, label_idx, unlabel_idx = split(
X=self.__X,
y=self.__y,
test_ratio=0.3,
initial_label_rate=0.05,
split_count=split_count,
all_class=True)
train_idx.pop()
test_idx.pop()
label_idx.pop()
unlabel_idx.pop()
# init the AlExperiment
try:
CrossValidationExperiment(X=self.__X,
Y=self.__y,
self_partition=False,
kfolds=5,
stopping_criteria=UnlabelSetEmpty(),
train_idx=train_idx,
test_idx=test_idx,
label_idx=label_idx,
unlabel_idx=unlabel_idx)
except ValueError as valExc:
assert ("Number of folds for inputs" in "{0}".format(valExc))
else:
raise Exception("Expected ValueError exception")
def setUp(self):
df = pd.read_csv('../resources/data/hmeq.csv') # import the dataset
df.dropna(axis=0, how='any', inplace=True)
df = pd.get_dummies(df, columns=['REASON', 'JOB'])
self.__X = df.drop(['BAD'], axis=1)
self.__y = df.filter(['BAD'], axis=1)
self.__train_idx, self.__test_idx, self.__label_idx, self.__unlabel_idx = split(
X=self.__X.to_numpy(),
y=self.__y['BAD'].to_numpy(),
test_ratio=0.3,
initial_label_rate=0.05,
split_count=1,
all_class=True)
self.__client = Client("tcp://192.168.2.100:8786")
def __init__(self,
client: Client,
X,
Y,
ml_technique,
scenario_type: AbstractScenario,
performance_metrics: [],
query_strategy: SingleLabelIndexQuery,
oracle: Oracle,
stopping_criteria: AbstractStopCriterion,
self_partition: bool,
kfolds: int = 1,
batch_size=1,
**kwargs):
"""
Parameters
----------
:param client: distributed.Client
:param X: array-like
Data matrix with [n_samples, n_features]
:param Y: array-like, optional
labels of given data [n_samples, n_labels] or [n_samples]
:param ml_technique
:param scenario_type: Sub-Type of AbstractScenario
Type of Active Learning scenario to use
:param performance_metrics: array-like of BaseMetrics elements
:param query_strategy: SinlgeLabelIndexQuery
:param oracle: Oracle
:param stopping_criteria: AbstractStopCriterion
:param self_partition: bool
:param kfolds: int, optional (default=1)
If self_partition is True Random split data k sets according to the extra parameters
-> test_ratio: float, optional (default=0.3)
Ratio of test set
-> initial_label_rate: float, optional (default=0.05)
Ratio of initial label set
e.g. Initial_labelset*(1-test_ratio)*n_samples
-> all_class: bool, optional (default=True)
Whether each split will contain at least one instance for each class.
If False, a totally random split will be performed.
If self_partition is False the following the following parameter must be specified
-> train_idx:
-> test_idx:
-> label_idx:
-> unlabel_idx:
:param kwargs: optional
Extra parameters
"""
self._client = client
if type(X) is da.core.Array:
self._X = X.persist()
else:
self._X = da.from_array(X, chunks=len(X) // 50).persist()
if isinstance(Y, da.core.Array):
self._Y = Y.persist()
else:
self._Y = da.from_array(Y, chunks=len(Y) // 50).persist()
# Persists the Dask Storage Structures
if client is not None and kwargs.pop("rebalance", False):
client.rebalance(self._X)
client.rebalance(self._Y)
check_X_y(self._X,
self._Y,
accept_sparse='csc',
multi_output=True,
distributed=False)
self._scenario_type = scenario_type
if self._scenario_type is None:
raise ValueError("required param 'scenario_type' can not be empty")
if not issubclass(self._scenario_type, AbstractScenario):
raise ValueError(
"the 'scenario_type' must be a subclass of 'AbstractScenario'")
if self_partition:
self._kfolds = kfolds
self._train_idx, self._test_idx, self._label_idx, self._unlabel_idx = split(
X=self._X,
y=self._Y,
test_ratio=kwargs.pop("test_ratio", 0.3),
initial_label_rate=kwargs.pop("initial_label_rate", 0.05),
split_count=self._kfolds,
all_class=kwargs.pop("all_class", True))
else:
train_idx = kwargs.pop("train_idx", None)
test_idx = kwargs.pop("test_idx", None)
label_idx = kwargs.pop("label_idx", None)
unlabel_idx = kwargs.pop("unlabel_idx", None)
if train_idx is None:
raise ValueError(
"required param 'train_idx' can not be empty ")
if test_idx is None:
raise ValueError("required param 'test_idx' can not be empty ")
if label_idx is None:
raise ValueError(
"required param 'label_idx' can not be empty ")
if unlabel_idx is None:
raise ValueError(
"required param 'unlabel_idx' can not be empty ")
num_inst_x, num_feat = da.shape(self._X)
num_inst_y, num_labels = da.shape(
self._Y) if len(da.shape(self._Y)) > 1 else (da.shape(
self._Y)[0], 1)
folds_train, num_inst_train = np.shape(train_idx)
folds_test, num_inst_test = np.shape(test_idx)
folds_labeled, num_inst_labeled = np.shape(label_idx)
folds_unlabeled, num_inst_unlabeled = np.shape(unlabel_idx)
if num_inst_x != num_inst_y:
raise ValueError(
"Different numbers of instances for inputs (x:%s, y:%s)" %
(num_inst_x, num_inst_y))
if folds_train != folds_test or folds_test != folds_labeled or folds_labeled != folds_unlabeled:
raise ValueError(
"Different numbers of folds for inputs (train_idx:%s, test_idx:%s "
"label_idx:%s, unlabel_idx:%s)" %
(folds_train, folds_test, folds_labeled, folds_unlabeled))
if kfolds != folds_test:
raise ValueError(
"Number of folds for inputs (train_idx:%s, test_idx:%s "
"label_idx:%s, unlabel_idx:%s) must be equals to kfolds:%s param"
% (folds_train, folds_test, folds_labeled, folds_unlabeled,
kfolds))
if num_inst_train + num_inst_test != num_inst_x:
raise ValueError(
"The sum of the number of instances for train_idx and test_idx must be equal to the "
"number of instances for x"
"(num_inst_x:%s, num_inst_train:%s num_inst_test:%s)" %
(num_inst_x, num_inst_train, num_inst_test))
if num_inst_labeled + num_inst_unlabeled != num_inst_train:
raise ValueError(
"The sum of the number of instances for label_idx and unlabel_idx must be equal to the "
"number of instances for train_idx"
"(num_inst_labeled:%s, num_inst_unlabeled:%s num_inst_unlabeled:%s)"
%
(num_inst_labeled, num_inst_unlabeled, num_inst_unlabeled))
self._kfolds = folds_train
self._train_idx = train_idx
self._test_idx = test_idx
self._label_idx = label_idx
self._unlabel_idx = unlabel_idx
self._ml_technique = ml_technique
if self._ml_technique is None:
raise ValueError("required param 'ml_technique' can not be empty")
self._performance_metrics = performance_metrics
if self._performance_metrics is None or len(
self._performance_metrics) == 0:
raise ValueError(
"required param 'performance_metric' can not be empty")
else:
for metric in self._performance_metrics:
if not isinstance(metric, BaseMetrics):
raise ValueError(
"the elements in 'performance_metrics' must be of type BaseMetrics"
)
self._query_strategy = query_strategy
if self._query_strategy is None:
raise ValueError(
"required param 'query_strategy' can not be empty")
self._oracle = oracle
if self._oracle is None:
raise ValueError("required param 'simOracle' can not be empty")
self._stopping_criteria = stopping_criteria
if self._stopping_criteria is None:
raise ValueError(
"required param 'stopping_criteria' can not be empty")
# Dynamically create the scenario Type given the arguments
importlib.import_module(self._scenario_type.__module__)
self._scenario = eval(self._scenario_type.__qualname__)(
X=self._X,
y=self._Y,
train_idx=self._train_idx[0],
test_idx=self._test_idx[0],
label_idx=copy.deepcopy(IndexCollection(self._label_idx[0])),
unlabel_idx=copy.deepcopy(IndexCollection(self._unlabel_idx[0])),
ml_technique=self._ml_technique,
performance_metrics=self._performance_metrics,
query_strategy=self._query_strategy,
oracle=self._oracle,
batch_size=batch_size)
class TestRegression(unittest.TestCase):
# logging.basicConfig(level=logging.DEBUG)
# Get the data
__X = np.random.choice(np.linspace(0, 20, 10000), size=200, replace=False).reshape(-1, 1)
__y = np.sin(__X) + np.random.normal(scale=0.3, size=__X.shape)
# assembling initial training set
__train_idx, __test_idx, __label_idx, __unlabel_idx = split(
X=__X,
y=__y,
test_ratio=0.3,
initial_label_rate=0.05,
split_count=1,
all_class=True)
# defining the kernel for the Gaussian process
__ml_technique = GaussianProcessRegressor(
kernel=RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \
+ WhiteKernel(noise_level=1, noise_level_bounds=(1e-10, 1e+1)))
def test_One_iteration(self):
experiment = HoldOutExperiment(
client=None,
X=self.__X,
Y=self.__y,
scenario_type=PoolBasedSamplingScenario,
train_idx=self.__train_idx,
test_idx=self.__test_idx,
label_idx=self.__label_idx,
unlabel_idx=self.__unlabel_idx,
ml_technique=self.__ml_technique,
performance_metrics=[Mse(squared=True)],
query_strategy=QueryRegressionStd(),
oracle=SimulatedOracle(labels=self.__y),
stopping_criteria=MaxIteration(1),
self_partition=False
)
result = experiment.evaluate(verbose=False)
regressor = result[0].ml_technique
# plotting the initial estimation
with plt.style.context('seaborn-white'):
plt.figure(figsize=(14, 7))
x = np.linspace(0, 20, 1000)
pred, std = regressor.predict(x.reshape(-1, 1), return_std=True)
plt.plot(x, pred)
plt.fill_between(x, pred.reshape(-1, ) - std, pred.reshape(-1, ) + std, alpha=0.2)
plt.scatter(self.__X, self.__y, c='k')
plt.title('Initial estimation')
plt.show()
def test_fifteen_iteration(self):
experiment = HoldOutExperiment(
client=None,
X=self.__X,
Y=self.__y,
scenario_type=PoolBasedSamplingScenario,
train_idx=self.__train_idx,
test_idx=self.__test_idx,
label_idx=self.__label_idx,
unlabel_idx=self.__unlabel_idx,
ml_technique=self.__ml_technique,
performance_metrics=[Mse(squared=True)],
query_strategy=QueryRegressionStd(),
oracle=SimulatedOracle(labels=self.__y),
stopping_criteria=MaxIteration(15),
self_partition=False
)
result = experiment.evaluate(verbose=False)
regressor = result[0].ml_technique
# plotting the initial estimation
with plt.style.context('seaborn-white'):
plt.figure(figsize=(14, 7))
x = np.linspace(0, 20, 1000)
pred, std = regressor.predict(x.reshape(-1, 1), return_std=True)
plt.plot(x, pred)
plt.fill_between(x, pred.reshape(-1, ) - std, pred.reshape(-1, ) + std, alpha=0.2)
plt.scatter(self.__X, self.__y, c='k')
plt.title('Initial estimation')
plt.show()
def test_five_iteration_batch_size(self):
experiment = HoldOutExperiment(
client=None,
X=self.__X,
Y=self.__y,
scenario_type=PoolBasedSamplingScenario,
train_idx=self.__train_idx,
test_idx=self.__test_idx,
label_idx=self.__label_idx,
unlabel_idx=self.__unlabel_idx,
ml_technique=self.__ml_technique,
performance_metrics=[Mse(squared=True)],
query_strategy=QueryRegressionStd(),
oracle=SimulatedOracle(labels=self.__y),
stopping_criteria=MaxIteration(15),
self_partition=False,
batch_size=5
)
result = experiment.evaluate(verbose=False)
regressor = result[0].ml_technique
# plotting the initial estimation
with plt.style.context('seaborn-white'):
plt.figure(figsize=(14, 7))
x = np.linspace(0, 20, 1000)
pred, std = regressor.predict(x.reshape(-1, 1), return_std=True)
plt.plot(x, pred)
plt.fill_between(x, pred.reshape(-1, ) - std, pred.reshape(-1, ) + std, alpha=0.2)
plt.scatter(self.__X, self.__y, c='k')
plt.title('Initial estimation')
plt.show()
def test_keras_digits_recognition_active_learning(self):
# load the data - it returns 2 tuples of digits & labels - one for
(x_train, y_train), (x_test, y_test) = mnist.load_data()
batch_size = 1024
num_classes = 10
epochs = 3
# input image dimensions
img_rows, img_cols = 28, 28
# display 14 random images from the training set
np.random.seed(123)
rand_14 = np.random.randint(0, x_train.shape[0], 14)
sample_digits = x_train[rand_14]
sample_labels = y_train[rand_14]
num_rows, num_cols = 2, 7
f, ax = plt.subplots(num_rows,
num_cols,
figsize=(12, 5),
gridspec_kw={
'wspace': 0.03,
'hspace': 0.01
},
squeeze=True)
for r in range(num_rows):
for c in range(num_cols):
image_index = r * 7 + c
ax[r, c].axis("off")
ax[r, c].imshow(sample_digits[image_index], cmap='gray')
ax[r, c].set_title('No. %d' % sample_labels[image_index])
plt.show()
plt.close()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
ml_technique = Sequential()
ml_technique.add(
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
ml_technique.add(Conv2D(64, (3, 3), activation='relu'))
ml_technique.add(MaxPooling2D(pool_size=(2, 2)))
ml_technique.add(Dropout(0.25))
ml_technique.add(Flatten())
ml_technique.add(Dense(128, activation='relu'))
ml_technique.add(Dropout(0.5))
ml_technique.add(Dense(num_classes, activation='softmax'))
ml_technique.compile(optimizer='Adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
X = np.concatenate((x_train, x_test))
y = np.concatenate((y_train, y_test))
train_idx, test_idx, label_idx, unlabel_idx = split(
X=X,
y=y,
test_ratio=0.3,
initial_label_rate=0.05,
split_count=1,
all_class=True)
# convert to indexed collection
train_idx = IndexCollection(train_idx[0])
test_idx = IndexCollection(test_idx[0])
label_idx = IndexCollection(label_idx[0])
unlabel_idx = IndexCollection(unlabel_idx[0])
# Define the active learning components
stopping_criteria = MaxIteration(10)
query_strategy = QueryLeastConfidentSampling()
oracle = SimulatedOracle(labels=y)
start_time = time.time()
experimentState = State(
round=0,
train_idx=train_idx,
test_idx=test_idx,
init_L=label_idx,
init_U=unlabel_idx,
performance_metrics=[metric for metric in ["loss", "accuracy"]],
verbose=True)
while not stopping_criteria.is_stop() and len(unlabel_idx) > 0:
label_x = X[label_idx.index, :]
label_y = y[label_idx.index]
test_x = X[test_idx, :]
test_y = y[test_idx]
# Train and evaluate Model over the labeled instances
ml_technique.fit(label_x,
label_y,
batch_size=batch_size,
epochs=epochs,
verbose=True,
validation_data=(test_x, test_y))
# predict the results over the labeled test instances
label_pred = ml_technique.predict_classes(test_x)
# performance calc for all metrics
label_perf = []
score = ml_technique.evaluate(x_test, y_test, verbose=1)
label_perf.append({"name": "loss", "value": score[0]})
label_perf.append({"name": "accuracy", "value": score[1]})
# use the query strategy for selecting the indexes
select_ind = query_strategy.select(X=X,
y=y,
label_index=label_idx,
unlabel_index=unlabel_idx,
batch_size=batch_size,
model=ml_technique,
client=self.__client)
# show label values
oracle.query(instances=X[select_ind], indexes=select_ind)
# update label and unlabel instaces
label_idx.update(select_ind)
unlabel_idx.difference_update(select_ind)
# save intermediate results
experimentState.add_state(
StateItem(select_index=select_ind,
performance_metrics=[
metric['name'] for metric in label_perf
],
performance=label_perf))
# update stopping_criteria
stopping_criteria.update_information(experimentState)
end_time = time.time() - start_time
print(end_time)
query_analyser = ExperimentAnalyserFactory.experiment_analyser(
performance_metrics=[metric for metric in ["loss", "accuracy"]],
method_name=query_strategy.query_function_name,
method_results=[experimentState],
type="queries")
# get a brief description of the experiment
query_analyser.plot_learning_curves(
title='Active Learning experiment results')
def test_custom_activeLearning_keras(self):
batch_size = 5
epochs = 20
# partition the data
train_idx, test_idx, label_idx, unlabel_idx = split(
X=self.__X,
y=self.__y,
test_ratio=0.3,
initial_label_rate=0.05,
split_count=1,
all_class=True)
# convert to indexed collection
train_idx = IndexCollection(train_idx[0])
test_idx = IndexCollection(test_idx[0])
label_idx = IndexCollection(label_idx[0])
unlabel_idx = IndexCollection(unlabel_idx[0])
# Create the model
ml_technique = Sequential()
ml_technique.add(Dense(input_dim=30, units=30))
ml_technique.add(Dense(input_dim=30, units=30))
ml_technique.add(Dense(input_dim=30, units=2))
ml_technique.add(Activation('softmax'))
ml_technique.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Define the active learning components
stopping_criteria = MaxIteration(10)
query_strategy = QueryLeastConfidentSampling()
performance_metrics = [
Accuracy(), F1(average='weighted'),
HammingLoss()
]
oracle = SimulatedOracle(labels=self.__y)
start_time = time.time()
experimentState = State(round=0,
train_idx=train_idx,
test_idx=test_idx,
init_L=label_idx,
init_U=unlabel_idx,
performance_metrics=[
metric.metric_name
for metric in performance_metrics
],
verbose=True)
while not stopping_criteria.is_stop() and len(unlabel_idx) > 0:
label_x = self.__X[label_idx.index, :]
label_y = self.__y[label_idx.index]
test_x = self.__X[test_idx, :]
test_y = self.__y[test_idx]
# Train and evaluate Model over the labeled instances
ml_technique.fit(label_x,
label_y,
batch_size=batch_size,
epochs=epochs,
verbose=True)
# predict the results over the labeled test instances
label_pred = ml_technique.predict_classes(test_x)
# performance calc for all metrics
label_perf = []
for metric in performance_metrics:
value = metric.compute(y_true=test_y, y_pred=label_pred)
label_perf.append({"name": metric.metric_name, "value": value})
# use the query strategy for selecting the indexes
select_ind = query_strategy.select(X=self.__X,
y=self.__y,
label_index=label_idx,
unlabel_index=unlabel_idx,
batch_size=batch_size,
model=ml_technique,
client=self.__client)
# show label values
oracle.query(instances=self.__X[select_ind], indexes=select_ind)
# update label and unlabel instaces
label_idx.update(select_ind)
unlabel_idx.difference_update(select_ind)
# save intermediate results
experimentState.add_state(
StateItem(select_index=select_ind,
performance_metrics=[
metric['name'] for metric in label_perf
],
performance=label_perf))
# update stopping_criteria
stopping_criteria.update_information(experimentState)
end_time = time.time() - start_time
print(end_time)
query_analyser = ExperimentAnalyserFactory.experiment_analyser(
performance_metrics=[
metric.metric_name for metric in performance_metrics
],
method_name=query_strategy.query_function_name,
method_results=[experimentState],
type="queries")
# get a brief description of the experiment
query_analyser.plot_learning_curves(
title='Active Learning experiment results')