def LF_experiment(num_task_1_data, rep):
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {"kwargs": {"max_depth": 30}}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleAverage
default_decider_kwargs = {}
progressive_learner = ProgressiveLearner(
default_transformer_class=default_transformer_class,
default_transformer_kwargs=default_transformer_kwargs,
default_voter_class=default_voter_class,
default_voter_kwargs=default_voter_kwargs,
default_decider_class=default_decider_class)
X_train_task0, y_train_task0 = generate_gaussian_parity(n=num_task_1_data,
angle_params=0,
acorn=1)
X_train_task1, y_train_task1 = generate_gaussian_parity(n=100,
angle_params=10,
acorn=1)
X_test_task0, y_test_task0 = generate_gaussian_parity(n=10000,
angle_params=0,
acorn=2)
progressive_learner.add_task(
X_train_task0,
y_train_task0,
num_transformers=10,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={"classes": np.unique(y_train_task0)})
llf_task = progressive_learner.predict(X_test_task0, task_id=0)
single_task_accuracy = np.nanmean(llf_task == y_test_task0)
single_task_error = 1 - single_task_accuracy
progressive_learner.add_transformer(X=X_train_task1,
y=y_train_task1,
transformer_data_proportion=1,
num_transformers=10,
backward_task_ids=[0])
llf_task = progressive_learner.predict(X_test_task0, task_id=0)
double_task_accuracy = np.nanmean(llf_task == y_test_task0)
double_task_error = 1 - double_task_accuracy
if double_task_error == 0 or single_task_error == 0:
te = 1
else:
te = (single_task_error + 1e-6) / (double_task_error + 1e-6)
df = pd.DataFrame()
df['te'] = [te]
print('n = {}, te = {}'.format(num_task_1_data, te))
file_to_save = 'result/' + str(num_task_1_data) + '_' + str(
rep) + '.pickle'
with open(file_to_save, 'wb') as f:
pickle.dump(df, f)
def LF_experiment(angle, reps=1, ntrees=10, acorn=None):
errors = np.zeros(2)
for rep in range(reps):
print("Starting Rep {} of Angle {}".format(rep, angle))
X_base_train, y_base_train = generate_gaussian_parity(n=100,
angle_params=0,
acorn=rep)
X_base_test, y_base_test = generate_gaussian_parity(n=10000,
angle_params=0,
acorn=rep)
X_rotated_train, y_rotated_train = generate_gaussian_parity(
n=100, angle_params=angle, acorn=rep)
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {"kwargs": {"max_depth": 10}}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleAverage
default_decider_kwargs = {}
progressive_learner = ProgressiveLearner(
default_transformer_class=default_transformer_class,
default_transformer_kwargs=default_transformer_kwargs,
default_voter_class=default_voter_class,
default_voter_kwargs=default_voter_kwargs,
default_decider_class=default_decider_class)
progressive_learner.add_task(
X_base_train,
y_base_train,
num_transformers=ntrees,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={"classes": np.unique(y_base_train)})
base_predictions_test = progressive_learner.predict(X_base_test,
task_id=0)
progressive_learner.add_transformer(X=X_rotated_train,
y=y_rotated_train,
transformer_data_proportion=1,
num_transformers=10,
backward_task_ids=[0])
all_predictions_test = progressive_learner.predict(X_base_test,
task_id=0)
errors[1] = errors[1] + (1 -
np.mean(all_predictions_test == y_base_test))
errors[0] = errors[0] + (1 -
np.mean(base_predictions_test == y_base_test))
errors = errors / reps
print("Errors For Angle {}: {}".format(angle, errors))
with open('results/angle_' + str(angle) + '.pickle', 'wb') as f:
pickle.dump(errors, f, protocol=2)
def LF_experiment(data_x, data_y, angle, model, granularity, reps=1, ntrees=29, acorn=None):
if acorn is not None:
np.random.seed(acorn)
errors = np.zeros(2)
with tf.device('/gpu:'+str(int(angle // granularity) % 4)):
for rep in range(reps):
train_x1, train_y1, train_x2, train_y2, test_x, test_y = cross_val_data(data_x, data_y, total_cls=10)
#change data angle for second task
tmp_data = train_x2.copy()
_tmp_ = np.zeros((32,32,3), dtype=int)
total_data = tmp_data.shape[0]
for i in range(total_data):
tmp_ = image_aug(tmp_data[i],angle)
tmp_data[i] = tmp_
if model == "uf":
train_x1 = train_x1.reshape((train_x1.shape[0], train_x1.shape[1] * train_x1.shape[2] * train_x1.shape[3]))
tmp_data = tmp_data.reshape((tmp_data.shape[0], tmp_data.shape[1] * tmp_data.shape[2] * tmp_data.shape[3]))
test_x = test_x.reshape((test_x.shape[0], test_x.shape[1] * test_x.shape[2] * test_x.shape[3]))
if model == "dnn":
default_transformer_class = NeuralClassificationTransformer
network = keras.Sequential()
network.add(layers.Conv2D(filters=16, kernel_size=(3, 3), activation='relu', input_shape=np.shape(train_x1)[1:]))
network.add(layers.BatchNormalization())
network.add(layers.Conv2D(filters=32, kernel_size=(3, 3), strides = 2, padding = "same", activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Conv2D(filters=64, kernel_size=(3, 3), strides = 2, padding = "same", activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Conv2D(filters=128, kernel_size=(3, 3), strides = 2, padding = "same", activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Conv2D(filters=254, kernel_size=(3, 3), strides = 2, padding = "same", activation='relu'))
network.add(layers.Flatten())
network.add(layers.BatchNormalization())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Dense(units=10, activation = 'softmax'))
default_transformer_kwargs = {"network" : network,
"euclidean_layer_idx" : -2,
"num_classes" : 10,
"optimizer" : keras.optimizers.Adam(3e-4)
}
default_voter_class = KNNClassificationVoter
default_voter_kwargs = {"k" : int(np.log2(len(train_x1)))}
default_decider_class = SimpleArgmaxAverage
elif model == "uf":
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {"kwargs" : {"max_depth" : 30}}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleArgmaxAverage
progressive_learner = ProgressiveLearner(default_transformer_class = default_transformer_class,
default_transformer_kwargs = default_transformer_kwargs,
default_voter_class = default_voter_class,
default_voter_kwargs = default_voter_kwargs,
default_decider_class = default_decider_class)
progressive_learner.add_task(
X = train_x1,
y = train_y1,
transformer_voter_decider_split = [0.67, 0.33, 0],
decider_kwargs = {"classes" : np.unique(train_y1)}
)
progressive_learner.add_transformer(
X = tmp_data,
y = train_y2,
transformer_data_proportion = 1,
backward_task_ids = [0]
)
llf_task1=progressive_learner.predict(test_x, task_id=0)
llf_single_task=progressive_learner.predict(test_x, task_id=0, transformer_ids=[0])
errors[1] = errors[1]+(1 - np.mean(llf_task1 == test_y))
errors[0] = errors[0]+(1 - np.mean(llf_single_task == test_y))
errors = errors/reps
print("Errors For Angle {}: {}".format(angle, errors))
with open('results/angle_'+str(angle)+'_'+model+'.pickle', 'wb') as f:
pickle.dump(errors, f, protocol = 2)
def LF_experiment(data_x,
data_y,
ntrees,
shift,
slot,
model,
num_points_per_task,
acorn=None):
df = pd.DataFrame()
shifts = []
slots = []
accuracies_across_tasks = []
train_times_across_tasks = []
inference_times_across_tasks = []
train_x_task0, train_y_task0, test_x_task0, test_y_task0 = cross_val_data(
data_x,
data_y,
num_points_per_task,
total_task=10,
shift=shift,
slot=slot)
if model == "dnn":
default_transformer_class = NeuralClassificationTransformer
network = keras.Sequential()
network.add(
layers.Conv2D(filters=16,
kernel_size=(3, 3),
activation='relu',
input_shape=np.shape(train_x_task0)[1:]))
network.add(
layers.Conv2D(filters=32,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(
layers.Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(
layers.Conv2D(filters=128,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(
layers.Conv2D(filters=254,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.Flatten())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.Dense(units=10, activation='softmax'))
default_transformer_kwargs = {
"network": network,
"euclidean_layer_idx": -2,
"num_classes": 10,
"optimizer": keras.optimizers.Adam(3e-4)
}
default_voter_class = KNNClassificationVoter
default_voter_kwargs = {"k": int(np.log2(num_points_per_task * .33))}
default_decider_class = SimpleArgmaxAverage
elif model == "uf":
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {"kwargs": {"max_depth": 30}}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleArgmaxAverage
progressive_learner = ProgressiveLearner(
default_transformer_class=default_transformer_class,
default_transformer_kwargs=default_transformer_kwargs,
default_voter_class=default_voter_class,
default_voter_kwargs=default_voter_kwargs,
default_decider_class=default_decider_class)
train_start_time = time.time()
progressive_learner.add_task(
X=train_x_task0,
y=train_y_task0,
num_transformers=1 if model == "dnn" else ntrees,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={"classes": np.unique(train_y_task0)})
train_end_time = time.time()
inference_start_time = time.time()
task_0_predictions = progressive_learner.predict(test_x_task0, task_id=0)
inference_end_time = time.time()
shifts.append(shift)
slots.append(slot)
accuracies_across_tasks.append(np.mean(task_0_predictions == test_y_task0))
train_times_across_tasks.append(train_end_time - train_start_time)
inference_times_across_tasks.append(inference_end_time -
inference_start_time)
for task_ii in range(1, 20):
train_x, train_y, _, _ = cross_val_data(data_x,
data_y,
num_points_per_task,
total_task=10,
shift=shift,
slot=slot,
task=task_ii)
print("Starting Task {} For Fold {} For Slot {}".format(
task_ii, shift, slot))
train_start_time = time.time()
progressive_learner.add_transformer(
X=train_x,
y=train_y,
transformer_data_proportion=1,
num_transformers=1 if model == "dnn" else ntrees,
backward_task_ids=[0])
train_end_time = time.time()
inference_start_time = time.time()
task_0_predictions = progressive_learner.predict(test_x_task0,
task_id=0)
inference_end_time = time.time()
shifts.append(shift)
slots.append(slot)
accuracies_across_tasks.append(
np.mean(task_0_predictions == test_y_task0))
train_times_across_tasks.append(train_end_time - train_start_time)
inference_times_across_tasks.append(inference_end_time -
inference_start_time)
print("Accuracy Across Tasks: {}".format(accuracies_across_tasks))
print("Train Times Across Tasks: {}".format(train_times_across_tasks))
print("Inference Times Across Tasks: {}".format(
inference_times_across_tasks))
df['data_fold'] = shifts
df['slot'] = slots
df['accuracy'] = accuracies_across_tasks
df['train_times'] = train_times_across_tasks
df['inference_times'] = inference_times_across_tasks
file_to_save = 'result/' + model + str(ntrees) + '_' + str(
shift) + '_' + str(slot) + '.pickle'
with open(file_to_save, 'wb') as f:
pickle.dump(df, f)
def L2_experiment(data_x,
data_y,
ntrees,
shift,
slot,
num_points_per_task,
task_num,
acorn=None):
# construct dataframes
df = pd.DataFrame()
shifts = []
slots = []
accuracies_across_tasks = []
# randomly separate the training and testing subsets
train_x_task0, train_y_task0, test_x_task0, test_y_task0 = cross_val_data(
data_x,
data_y,
num_points_per_task,
total_task=10,
shift=shift,
slot=slot)
# choose Uncertainty Forest as transformer
progressive_learner = ProgressiveLearner(
default_transformer_class=TreeClassificationTransformer,
default_transformer_kwargs={"kwargs": {
"max_depth": 30
}},
default_voter_class=TreeClassificationVoter,
default_voter_kwargs={},
default_decider_class=SimpleArgmaxAverage,
)
# training process
progressive_learner.add_task(
X=train_x_task0,
y=train_y_task0,
num_transformers=ntrees,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={"classes": np.unique(train_y_task0)},
)
# testing process
task_0_predictions = progressive_learner.predict(test_x_task0, task_id=0)
# record results
shifts.append(shift)
slots.append(slot)
accuracies_across_tasks.append(np.mean(task_0_predictions == test_y_task0))
# repeating the tasks for task_num times
for task_ii in range(1, task_num):
# randomly separate the training and testing subsets
train_x, train_y, _, _ = cross_val_data(
data_x,
data_y,
num_points_per_task,
total_task=10,
shift=shift,
slot=slot,
task=task_ii,
)
# training process
progressive_learner.add_transformer(
X=train_x,
y=train_y,
transformer_data_proportion=1,
num_transformers=ntrees,
backward_task_ids=[0],
)
# testing process
task_0_predictions = progressive_learner.predict(test_x_task0,
task_id=0)
# record results
shifts.append(shift)
slots.append(slot)
accuracies_across_tasks.append(
np.mean(task_0_predictions == test_y_task0))
# finalize dataframes
df["data_fold"] = shifts
df["slot"] = slots
df["accuracy"] = accuracies_across_tasks
# save results
return df
def odif_experiment(angle, data_x, data_y, reps=1, ntrees=29, acorn=None):
if acorn is not None:
np.random.seed(acorn)
errors = np.zeros(2)
for _ in range(reps):
train_x1, train_y1, train_x2, train_y2, test_x, test_y = cross_val_data(
data_x, data_y, total_cls=10
)
# change data angle for second task
tmp_data = train_x2.copy()
total_data = tmp_data.shape[0]
for i in range(total_data):
tmp_ = image_aug(tmp_data[i], angle)
tmp_data[i] = tmp_
train_x1 = train_x1.reshape(
(
train_x1.shape[0],
train_x1.shape[1] * train_x1.shape[2] * train_x1.shape[3],
)
)
tmp_data = tmp_data.reshape(
(
tmp_data.shape[0],
tmp_data.shape[1] * tmp_data.shape[2] * tmp_data.shape[3],
)
)
test_x = test_x.reshape(
(test_x.shape[0], test_x.shape[1] * test_x.shape[2] * test_x.shape[3])
)
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {"kwargs": {"max_depth": 30}}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleArgmaxAverage
progressive_learner = ProgressiveLearner(
default_transformer_class=default_transformer_class,
default_transformer_kwargs=default_transformer_kwargs,
default_voter_class=default_voter_class,
default_voter_kwargs=default_voter_kwargs,
default_decider_class=default_decider_class,
)
progressive_learner.add_task(
X=train_x1,
y=train_y1,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={"classes": np.unique(train_y1)},
)
progressive_learner.add_transformer(
X=tmp_data, y=train_y2, transformer_data_proportion=1, backward_task_ids=[0]
)
llf_task1 = progressive_learner.predict(test_x, task_id=0)
llf_single_task = progressive_learner.predict(
test_x, task_id=0, transformer_ids=[0]
)
errors[1] = errors[1] + (1 - np.mean(llf_task1 == test_y))
errors[0] = errors[0] + (1 - np.mean(llf_single_task == test_y))
errors = errors / reps
return errors
def single_experiment(x, y, y_speaker, ntrees=19, model='uf', shuffle=False):
num_tasks = 6
num_points_per_task = 3000 / num_tasks
speakers = ['g', 'j', 'l', 'n', 't', 'y']
accuracies_across_tasks = []
if model == 'dnn':
x_all = x
y_all = y
default_transformer_class = NeuralClassificationTransformer
network = keras.Sequential()
network.add(
layers.Conv2D(filters=16,
kernel_size=(3, 3),
activation='relu',
input_shape=np.shape(x_all)[1:]))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=32,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=128,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=254,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.Flatten())
network.add(layers.BatchNormalization())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Dense(units=10, activation='softmax'))
default_transformer_kwargs = {
"network": network,
"euclidean_layer_idx": -2,
"loss": "categorical_crossentropy",
"optimizer": Adam(3e-4),
"fit_kwargs": {
"epochs": 100,
"callbacks": [EarlyStopping(patience=5, monitor="val_loss")],
"verbose": False,
"validation_split": 0.33,
"batch_size": 32,
},
}
default_voter_class = KNNClassificationVoter
default_voter_kwargs = {"k": int(np.log2(num_points_per_task))}
default_decider_class = SimpleArgmaxAverage
elif model == 'uf':
x_all = x.reshape(3000, -1)
y_all = y
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {"kwargs": {"max_depth": 30}}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleArgmaxAverage
if shuffle:
np.random.shuffle(speakers)
else:
pass
for j, task0_speaker in enumerate(speakers):
progressive_learner = ProgressiveLearner(
default_transformer_class=default_transformer_class,
default_transformer_kwargs=default_transformer_kwargs,
default_voter_class=default_voter_class,
default_voter_kwargs=default_voter_kwargs,
default_decider_class=default_decider_class)
index = np.where(y_speaker == task0_speaker)
x_task0 = x_all[index]
y_task0 = y_all[index]
train_x_task0, test_x_task0, train_y_task0, test_y_task0 = train_test_split(
x_task0, y_task0, test_size=0.25)
progressive_learner.add_task(
X=train_x_task0,
y=train_y_task0,
task_id=0,
num_transformers=1 if model == "dnn" else ntrees,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={"classes": np.unique(train_y_task0)},
)
task_0_predictions = progressive_learner.predict(test_x_task0,
task_id=0)
accuracies_across_tasks.append(
np.mean(task_0_predictions == test_y_task0))
for k, contribute_speaker in enumerate(speakers):
if k == j:
pass
else:
index = np.where(y_speaker == contribute_speaker)
x_train = x_all[index]
y_train = y_all[index]
progressive_learner.add_transformer(
X=x_train,
y=y_train,
transformer_data_proportion=1,
num_transformers=1 if model == "dnn" else ntrees,
backward_task_ids=[0],
)
task_0_predictions = progressive_learner.predict(test_x_task0,
task_id=0)
accuracies_across_tasks.append(
np.mean(task_0_predictions == test_y_task0))
return accuracies_across_tasks