def nnet(self, trX, trY, previous=None, epochs=10, batch_size=100):
# create a new convnet, basing the weights on those of the previous net if possible
cnn = ConvolutionalNeuralNetwork()
if not previous:
cnn.w1 = cnn.init_weights((32, 1, 3, 3))
cnn.w2 = cnn.init_weights((64, 32, 3, 3))
cnn.w3 = cnn.init_weights((128, 64, 3, 3))
cnn.w4 = cnn.init_weights((128 * 3 * 3, 625))
cnn.wo = cnn.init_weights((625, 2))
else:
# np.copy and theano.tensor.copy don't create a fully disconnected deep copy, so we cry a little inside and use a temporary file :'(
filename = "tmp.txt"
previous.save_data(filename, previous.w1)
cnn.w1 = cnn.load_data(filename, (32, 1, 3, 3))
previous.save_data(filename, previous.w2)
cnn.w2 = cnn.load_data(filename, (64, 32, 3, 3))
previous.save_data(filename, previous.w3)
cnn.w3 = cnn.load_data(filename, (128, 64, 3, 3))
previous.save_data(filename, previous.w4)
cnn.w4 = cnn.load_data(filename, (128 * 3 * 3, 625))
previous.save_data(filename, previous.wo)
cnn.wo = cnn.load_data(filename, (625, 2))
os.remove(filename)
cnn.create_model_functions()
cnn.train_net(epochs, batch_size, trX=trX, trY=trY)
return cnn
def __init__(self):
super(ConvWrapper, self).__init__()
self.net = ConvolutionalNeuralNetwork()
self.net.initialize_mnist()
def calculate_catastrophic_interference(num_tasks,
exclude_start,
exclude_end,
top_layer="cnn",
save_figs=False,
verbose=False,
epochs=20,
batch_size=100):
excluded = range(exclude_start, exclude_end)
task_nums = [i for i in range(num_tasks) if i not in excluded]
start = time.time()
cnn = ConvolutionalNeuralNetwork()
cnn.initialize_mnist()
# cnn.trX = cnn.trX[:int(len(cnn.trX)*.2)]
# cnn.trY = cnn.trY[:int(len(cnn.trY)*.2)]
# cnn.teX = cnn.teX[:int(len(cnn.teX)*.2)]
# cnn.teY = cnn.teY[:int(len(cnn.teY)*.2)]
cnn.trX, cnn.trY, trXE, trYE = split_dataset(excluded, cnn.trX, cnn.trY)
cnn.teX, cnn.teY, teXE, teYE = split_dataset(excluded, cnn.teX, cnn.teY)
cnn.create_model_functions()
colors = [
"#00FF00", "#0000FF", "#00FFFF", "#FFFF00", "#FF00FF", "#000000",
"#888888", "#FF8800", "#88FF00", "#FF0088"
]
print("\nTraining on tasks {0}, excluding tasks {1}".format(
task_nums, excluded))
base_accuracies = train_per_task(cnn, num_tasks, verbose, epochs,
batch_size)
end = time.time()
print("Initial training: {0:0.02f}sec".format(end - start))
# base model, trained without excluded tasks
# (which are then added back in one of the three top-layer models)
# if save_figs:
# for t in task_nums:
# plt.plot(np.arange(0, epochs), accuracies[t], color = colors[t])
# plt.plot(np.arange(0, epochs), accuracies["total"], color = "#FF0000", marker = "o")
# plt.axis([0, epochs-1, 0, 1])
# plt.xlabel("Epoch")
# plt.ylabel("Accuracy")
# plt.title("Model Accuracy")
# plt.legend(["Task {0}".format(t) for t in task_nums]+["Total"], loc = "lower right")
# plt.savefig("figures/trained on {0}, excluded {1}.png".format(task_nums, excluded), bbox_inches = "tight")
# plt.close()
total_trX = np.concatenate((cnn.trX, trXE), axis=0)
total_trY = np.concatenate((cnn.trY, trYE), axis=0)
total_teX = np.concatenate((cnn.teX, teXE), axis=0)
total_teY = np.concatenate((cnn.teY, teYE), axis=0)
num_chunks = 20
trA = np.concatenate([
cnn.activate(total_trX[(len(total_trX) / num_chunks *
i):(len(total_trX) / num_chunks * (i + 1))])
for i in range(num_chunks)
])
teA = cnn.activate(total_teX)
trC = np.argmax(total_trY, axis=1)
teC = np.argmax(total_teY, axis=1)
# convolutional neural network
if "cnn" in top_layer:
print(
"\nRetraining convolutional neural network on all tasks after excluding {0} from initial training"
.format(excluded))
start = time.time()
# fit model with data
cnn_accs = train_new_tasks(cnn, total_trX, total_trY, total_teX,
total_teY, num_tasks, verbose, epochs,
batch_size)
end = time.time()
print("ConvNet Retraining: {0:0.02f}sec".format(end - start))
# show accuracy improvement from additional model layer
print(
"[ConvNet(exclusion)] Testing data accuracy: {0:0.04f}"
.format(base_accuracies["total"][-1]))
print(
"[ConvNet(exclusion)+ConvNet(all)] Testing data accuracy: {0:0.04f}"
.format(cnn_accs["total"][-1]))
print(
"[(CN(E)+CN(A))-CN(E)] Accuracy improvement: {0:0.04f}"
.format(cnn_accs["total"][-1] - base_accuracies["total"][-1]))
# generate and save accuracy figures
if save_figs:
for t in range(num_tasks):
plt.plot(np.arange(0, epochs), cnn_accs[t], color=colors[t])
plt.plot(np.arange(0, epochs),
cnn_accs["total"],
color="#FF0000",
marker="o")
plt.legend(["Task {0}".format(t) for t in task_nums] + ["Total"],
loc="lower right")
plt.axis([0, epochs - 1, 0, 1])
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.title("Model Accuracy")
plt.savefig(
"figures/trained on {0}, excluded {1}, then retrained on all.png"
.format(task_nums, excluded),
bbox_inches="tight")
plt.close()
# efficient lifelong learning algorithm
if "ella" in top_layer:
print(
"\nTraining efficient lifelong learning algorithm on all tasks after excluding {0} from convnet training"
.format(excluded))
start = time.time()
# fit model with data
ella = ELLA(d=625,
k=5,
base_learner=LogisticRegression,
base_learner_kwargs={"tol": 10**-2},
mu=10**-3)
for task in range(num_tasks):
ella.fit(trA, binarize(trC, task), task)
predictions = np.argmax(np.asarray(
[ella.predict_logprobs(teA, i) for i in range(ella.T)]),
axis=0)
ella_acc = np.mean(predictions == teC)
end = time.time()
print("ELLA: {0:0.02f}sec".format(end - start))
# show accuracy improvement from additional model layer
print(
"[ConvNet] Testing data accuracy: {0:0.04f}"
.format(base_accuracies["total"][-1]))
print(
"[ConvNet+ELLA] Testing data accuracy: {0:0.04f}"
.format(ella_acc))
print(
"[(CN+ELLA)-CN] Accuracy improvement: {0:0.04f}"
.format(ella_acc - base_accuracies["total"][-1]))
# generate and save accuracy figures
if save_figs:
pass # need to generate per-task or per-epoch accuracies to have a good visualization
# logistic regression model
if "lr" in top_layer:
print(
"\nTraining logistic regression model on all tasks after excluding {0} from convnet training"
.format(excluded))
start = time.time()
# fit model with data
lr = LogisticRegression()
lr.fit(trA, trC)
logreg_accs = find_model_task_accuracies(lr, num_tasks, teA, teC)
end = time.time()
print("Logistic Regression: {0:0.02f}sec".format(end - start))
# show accuracy improvement from additional model layer
print(
"[ConvNet] Testing data accuracy: {0:0.04f}"
.format(base_accuracies["total"][-1]))
print(
"[ConvNet+LogReg] Testing data accuracy: {0:0.04f}"
.format(logreg_accs["total"]))
print(
"[(CN+LR)-CN] Accuracy improvement: {0:0.04f}"
.format(logreg_accs["total"] - base_accuracies["total"][-1]))
if verbose:
print(
"\nLogistic regression model accuracies after exclusion training:"
)
for key, value in logreg_accs.items():
print("Task: {0}, accuracy: {1:0.04f}".format(key, value))
# generate and save accuracy figures
if save_figs:
plotX = ["Task {0}".format(t)
for t in range(num_tasks)] + ["Total", "Average"]
plotY = [logreg_accs[t] for t in range(num_tasks)
] + [logreg_accs["total"],
np.mean(logreg_accs.values())]
plt.bar(range(len(plotX)), plotY)
plt.xticks(range(len(plotX)), plotX)
plt.title("Model Accuracy")
plt.savefig(
"figures/trained on {0}, excluded {1}, then logreg.png".format(
task_nums, excluded),
bbox_inches="tight")
plt.close()
# support vector classifier
if "svc" in top_layer:
print(
"\nTraining linear support vector classifier on all tasks after excluding {0} from convnet training"
.format(excluded))
start = time.time()
# fit model with data
svc = LinearSVC()
svc.fit(trA, trC)
svc_accs = find_model_task_accuracies(svc, num_tasks, teA, teC)
end = time.time()
print("Support Vector Classifier: {0:0.02f}sec".format(end - start))
# show accuracy improvement from additional model layer
print(
"[ConvNet] Testing data accuracy: {0:0.04f}"
.format(base_accuracies["total"][-1]))
print(
"[ConvNet+SVC] Testing data accuracy: {0:0.04f}"
.format(svc_accs["total"]))
print(
"[(CN+SVC)-CN] Accuracy improvement: {0:0.04f}"
.format(svc_accs["total"] - base_accuracies["total"][-1]))
if verbose:
print(
"\nSupport vector classifier accuracies after exclusion training:"
)
for key, value in svc_accs.items():
print("Task: {0}, accuracy: {1:0.04f}".format(key, value))
# generate and save accuracy figures
if save_figs:
plotX = ["Task {0}".format(t)
for t in range(num_tasks)] + ["Total", "Average"]
plotY = [svc_accs[t] for t in range(num_tasks)
] + [svc_accs["total"],
np.mean(svc_accs.values())]
plt.bar(range(len(plotX)), plotY)
plt.xticks(range(len(plotX)), plotX)
plt.title("Model Accuracy")
plt.savefig(
"figures/trained on {0}, excluded {1}, then svc.png".format(
task_nums, excluded),
bbox_inches="tight")
plt.close()
print("")
trX = np.load(data_dir + "trX.npy")
trX = trX[:, np.newaxis, :, :]
trY = np.load(data_dir + "trY.npy")
trY = np.concatenate((np.logical_not(trY).astype(np.int64), trY),
axis=1)
teX = np.load(data_dir + "teX.npy")
teX = teX[:, np.newaxis, :, :]
teY = np.load(data_dir + "teY.npy")
teY = np.concatenate((np.logical_not(teY).astype(np.int64), teY),
axis=1)
shape_dict = {
"w1": (32, 1, 3, 3),
"w2": (64, 32, 3, 3),
"w3": (128, 64, 3, 3),
"w4": (128 * 15 * 11, 11625),
"wo": (11625, 2)
}
else:
print("Dataset must be mnist, cifar, or office.")
sys.exit(1)
cnn = ConvolutionalNeuralNetwork().initialize_dataset(
trX, trY, teX, teY, shape_dict)
cnn.create_model_functions().train_net(epochs=10,
batch_size=32,
verbose=False)
print("Accuracy on the {0} dataset: {1:0.02f}%".format(
dataset,
cnn.calc_accuracy(teX, teY) * 100))
teX07, teY07, teX_8, teY_8 = remove_class(teX08, teY08, 8)
trY07 = one_hot(trY07, 8)
teY07 = one_hot(teY07, 8)
trY09 = one_hot(trY09, 10)
teY09 = one_hot(teY09, 10)
start = time.time()
shape_dict = {
"w1": (32, 1, 3, 3),
"w2": (64, 32, 3, 3),
"w3": (128, 64, 3, 3),
"w4": (128 * 3 * 3, 625),
"wo": (625, 8)
}
cnn = ConvolutionalNeuralNetwork().initialize_dataset(
trX07, trY07, teX07, teY07, shape_dict)
cnn.create_model_functions().train_net(epochs=10,
batch_size=100,
verbose=True)
end = time.time()
print("0-7: {0:0.02f}".format(end - start))
start = time.time()
shape_dict["wo"] = (625, 10)
cnn = ConvolutionalNeuralNetwork().initialize_dataset(
trX09, trY09, teX09, teY09, shape_dict)
cnn.create_model_functions().train_net(epochs=10,
batch_size=100,
verbose=True)
end = time.time()
print("0-9: {0:0.02f}".format(end - start))
