def do_testing(self, sess, cases, msg="Testing", bestk=None):
inputs = [case[0] for case in cases]
targets = [case[1] for case in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.error
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
display_interval=None,
testing=True)
if bestk is None:
print('%s Set Error = %f ' % (msg, testres))
print('%s Set Correct Classifications = %f %%' %
(msg, self.gethits(cases, sess)))
else:
print('%s Set Correct Classifications = %f %%' %
(msg, 100 * (testres / len(cases))))
return testres
def create_dendrogram(self, dendrogram_layers, number_of_cases, bestk):
self.reopen_current_session()
test_cases = self.caseman.get_testing_cases()
cases = test_cases[:number_of_cases]
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
grabvar_names = [grabvar for grabvar in self.grabvars]
grabvar_layers = []
for layer in dendrogram_layers:
for grabvar in grabvar_names:
if ('-'+str(layer)+'-out') in str(grabvar.name):
grabvar_layers.append(grabvar)
for grabvar in grabvar_layers:
features = []
labels = []
for index, case in enumerate(cases):
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(self.predictor,
[TFT.one_hot_to_int(list(v)) for v in targets], k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func, grabvar, self.probes,
session=self.current_session,
feed_dict=feeder, show_interval=self.show_interval,
step=self.global_training_step)
labels.append(TFT.bits_to_str(case[1]))
features.append(grabvals[index])
TFT.dendrogram(features, labels, title=grabvar.name)
def consider_early_stopping(self,
sess,
cases,
msg="Early Stopping",
bestk=None,
target_accuracy=None):
inputs = [case[0] for case in cases]
targets = [case[1] for case in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.error
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
display_interval=None,
testing=True)
print("\n CONSIDER EARLY STOPPING: \n")
if bestk is None:
print('%s Set Correct Classifications = %f %% \n' %
(msg, self.gethits(cases, sess)))
if self.gethits(cases, sess) > target_accuracy:
return True
else:
print('%s Set Correct Classifications = %f %% \n' %
(msg, 100 * (testres / len(cases))))
if 100 * (testres / len(cases)) > target_accuracy:
return True
print("\n Target Accuracy NOT reached - continue: \n")
return False
def do_mapping(self, no_of_cases, defined_grab_vars=[], dendrogram_layers=[], bestk=None, dendrogram=False, labels=False):
self.reopen_current_session()
if len(defined_grab_vars) > 0:
for (layer, type) in defined_grab_vars:
if (layer-1, 'in') not in defined_grab_vars:
self.add_grabvar(layer-1, type)
if len(dendrogram_layers) > 0:
for layer in dendrogram_layers:
self.add_dendrogram_var(layer-1, 'out')
cases = self.get_random_cases(no_of_cases)
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
if bestk is not None:
self.test_func = self.gen_match_counter(self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
feeder = {self.input: inputs, self.target: targets}
testres, grabvals, _ = self.run_one_step(self.test_func, [self.grabvars, self.dendrogram_vars], self.probes, session=self.current_session,
feed_dict=feeder)
self.display_grabvars(grabvals[0], self.grabvars)
if dendrogram:
if labels:
input_den = [target for target in targets]
else:
input_den = [TFT.bits_to_str(case) for case in inputs]
for i, grabval in enumerate(grabvals[1]):
TFT.dendrogram(grabval, input_den, title=self.dendrogram_vars[i].name)
def do_mapping(self, map_layers, number_of_cases, bestk):
self.reopen_current_session()
test_cases = self.caseman.get_training_cases()
cases = test_cases[:number_of_cases]
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
grabvar_names = [grabvar for grabvar in self.grabvars]
grabvar_layers = []
for layer in map_layers:
for grabvar in grabvar_names:
if ('-'+str(layer)+'-out') in str(grabvar.name):
grabvar_layers.append(grabvar)
for grabvar in grabvar_layers:
features = []
for index, case in enumerate(cases):
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(self.predictor,
[TFT.one_hot_to_int(list(v)) for v in targets], k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func, grabvar, self.probes,
session=self.current_session,
feed_dict=feeder, show_interval=self.show_interval,
step=self.global_training_step)
features.append(grabvals[index])
TFT.hinton_plot(np.array(features), fig=PLT.figure(), title=grabvar.name + " Activation Levels")
new_target = np.array(targets)
TFT.hinton_plot(new_target, fig=PLT.figure(), title="Input Targets")
def do_testing(self, sess, cases, msg='Testing', bestk=None):
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.error
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
show_interval=None)
if bestk is None:
print('%s Set Error = %f ' % (msg, testres))
else:
print('%s Set Correct Classifications = %f %%' %
(msg, 100 * (testres / len(cases))))
return testres # self.error uses MSE, so this is a per-case value when bestk=None
def do_mapping(self, sess, cases, start=0, msg='Mapping', bestk=None):
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
results_hinton = sess.run([self.test_func, self.grabvars[0:start]],
feed_dict=feeder)
dendro_cases = self.caseman.get_training_cases()
dendro_inputs = [c[0] for c in dendro_cases]
dendro_targets = [c[1] for c in dendro_cases]
dendro_feeder = {
self.input: dendro_inputs,
self.target: dendro_targets
}
results_dendro = sess.run([self.test_func, self.grabvars[start:]],
feed_dict=dendro_feeder)
#print(dendro_dict)
self.display_grabvars(results_hinton[1],
self.grabvars[0:start],
step=1)
for t in targets:
print("True Label: ", t)
total_labels = []
total_targets = []
for j in range(1, len(results_dendro)):
dendro_dict = {}
for i in range(len(results_dendro[j])):
for k in range(len(results_dendro[j][i])):
inp = dendro_targets[k]
res = results_dendro[j][i][k]
if (type(inp) == int):
string = str(inp)
else:
string = TFT.bits_to_str(inp)
if (string not in dendro_dict):
dendro_dict[string] = [i for i in res]
dendro_labels = []
dendro_data = []
for key in dendro_dict.keys():
dendro_labels.append(key)
dendro_data.append(dendro_dict[key])
#print("Labels: ", dendro_labels)
#print("Targets: ", dendro_targets)
total_labels.append(dendro_labels)
total_targets.append(dendro_data)
self.display_dendrograms(total_targets, self.grabvars[start:],
total_labels, start)
def do_mapping(self, session, cases):
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
testFunc = self.gen_match_counter(self.output, [TFT.one_hot_to_int(list(v)) for v in targets], k=1)
_, vals, _ = self.run_one_step(grabbed_vars = [self.mapVars] + [self.dendrogramVars] + [self.grabVars], probed_vars = self.probes,
operators = testFunc, session = session,
feed_dict = feeder, showInterval = 0)
mapped_vals = vals[0]
dendrogram_vals = vals[1]
grabbed_vals = vals[2]
return mapped_vals, dendrogram_vals, grabbed_vals
def do_mapping(self, sess, cases, msg='Mapping', bestk=1):
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.error
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
error, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
show_interval=None)
return grabvals # self.error uses MSE, so this is a per-case value
def do_testing(self, sess, cases, epoch = 'test', msg='Testing', bestk=1):
inputs = [c[0] for c in cases]; targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
testFunc = self.gen_match_counter(self.output, [TFT.one_hot_to_int(list(v)) for v in targets], k = 1)
accuracy, _, _ = self.run_one_step(grabbed_vars = self.grabVars, probed_vars = self.probes,
operators = testFunc, session = sess,
feed_dict = feeder, showInterval = 0)
loss, _, _ = self.run_one_step(grabbed_vars=self.grabVars, probed_vars=self.probes,
operators=self.error, session=sess,
feed_dict=feeder, showInterval=0)
accuracy = 100 * (accuracy / len(cases))
if bestk is None:
print('Epoch: %s\t\t %s loss: N/A\t\t %s : %s %%' % (epoch, msg, msg, accuracy))
else:
print('Epoch: %s\t\t %s loss: %5.4f\t\t %s accuracy: %d %%' % (epoch, msg, loss, msg, accuracy))
return loss
def run_mapping(self, case_generator = None, mapBatchSize = 0, mapLayers = [], mapDendrograms = []):
self.mapBatchSize = mapBatchSize # Size of batch of cases used for a map test. 0 indicates no map test
self.mapLayers = mapLayers # List of layers(their indices) to be visualized during a map test
self.mapDendrograms = mapDendrograms # List of layers(their indices) whose activation patterns will be used to make dendrograms
self.reopen_current_session()
if self.mapBatchSize:
# either a chosen set of cases or a random subset of the training cases:
cases = case_generator() if case_generator else self.caseMan.get_mapping_cases(self.mapBatchSize)
# Add the monitored variables
self.add_mapvars()
self.add_dendrogramvars()
# Add all grabbed variables
for weight in self.displayWeights:
self.add_grabvar(weight, 'wgt')
for bias in self.displayBiases:
self.add_grabvar(bias, 'bias')
# run map test
mapvals, dendrovals, grabvals = self.do_mapping(session = self.current_session, cases = cases)
# Plotting
names = [x.name for x in self.mapVars]
for i, v in enumerate(mapvals):
if type(v) == np.ndarray and len(v.shape) > 1: # If v is a matrix, use hinton plotting
TFT.hinton_plot(v, fig=None, title='Activation pattern of layer ' + names[i])
if len(self.mapDendrograms) > 0:
names = [x.name for x in self.dendrogramVars]
if TFT.is_bit_vector(cases[0][0]):
labels = [TFT.bits_to_str(s[0]) for s in cases]
else:
labels = [TFT.one_hot_to_int(c[1]) for c in cases]
for (i, v) in enumerate(dendrovals):
TFT.dendrogram(v, labels, title = 'Dendrogram of ' + names[i])
if len(grabvals) > 0:
self.display_grabvars(grabvals, self.grabVars, step = self.globalTrainingStep)
# hold until a button is pressed
while (not PLT.waitforbuttonpress()):
pass
PLT.close('all')
def do_mapping(self, msg="Mapping", bestk=None, plot=""):
self.reopen_current_session()
cases = self.caseman.getMappingCases(self.mapBatchSize)
self.grabvars.clear()
helpers.add_grabvars(self, self.wantedMapGrabvars)
helpers.add_grabvars(self, self.dendrogramLayers)
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=self.current_session,
feed_dict=feeder,
show_interval=None)
if (self.mapplot == 'display'):
eval("TFT.display_matrix(testres)")
elif (self.mapplot == "hinton"):
eval("TFT.hinton_plot(testres)")
elif (plot == "dendrogram"):
eval("TFT.dendrogram(inputs, targets)")
# Dendrogram for each layer...
# print("grabvals")
# print(grabvals)
for hiddenactivation in grabvals[len(self.wantedMapGrabvars):]:
# index, type
TFT.dendrogram(hiddenactivation, targets, ax=None)
# TFT.dendrogram(feature, labels) --> mappe input og hidden activation patterns ... waht are the input, and what is the labels ? I see that labels are suppose to be string..
# TFT.dendrogram(testres, targets, ax=None) # ser ut til å funke, men er jo feile verdier ?
self.close_current_session(view=False)
def do_mapping(self, sess, cases, msg='Mapping', bestk=None):
# must reopen current session to work
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
show_interval=1)
print(np.shape(testres))
print(np.shape(grabvals))
def do_testing(self, sess, cases, msg='Testing', bestk=None, epoch=0):
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
#print("\n--- " + msg + " ---\n")
self.test_func = self.error
if bestk is not None:
if (self.loss_func == "sparse_softmax_cross_entropy"):
self.test_func = self.gen_match_counter(self.predictor,
[v for v in targets],
k=bestk)
else:
self.test_func = self.gen_match_counter(
self.predictor,
[TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
show_interval=None)
if bestk is None:
if epoch == 0:
epoch = self.global_training_step
if (type(testres) != np.float64 and len(testres) > 1):
batch_error = 0
for e in testres:
batch_error += e
batch_error = float(batch_error / len(testres))
print('%s Set Error = %f ' % (msg, batch_error))
else:
print('%s Set Error = %f ' % (msg, testres))
else:
print('%s Set Correct Classifications = %f %%' %
(msg, 100 * (testres / len(cases))))
return testres # self.error uses MSE, so this is a per-case value when bestk=None
error = self.do_testing(case_list, scenario="validation")
# print(step, ", error=", error)
self.validation_error_history.append((step, error))
def do_testing(self, case_list=None, scenario="testing", grabvars=[]):
if not grabvars: grabvars = self.grabvars
if scenario == "testing" and not case_list:
case_list = self.case_manager.get_testing_cases()
# print(case_list[0].input)
# print(case_list[0].target)
inputs = [case.input for case in case_list]
targets = [case.target for case in case_list]
pred_targets = [t[0] if len(t) == 1 else TFT.one_hot_to_int(t) for t in targets]
sess = self.sess
feeder = {self.input: inputs, self.target: targets}
if self.output_layer_size == 1: # one outout node, special case
correct_pred = tf.equal(tf.round(self.output), targets)
num_correct = tf.reduce_sum(tf.cast(correct_pred, tf.int32))
else:
correct_pred = tf.nn.in_top_k(tf.cast(self.output, tf.float32), tf.cast(pred_targets, tf.int32), self.config.topk)
num_correct = tf.reduce_sum(tf.cast(correct_pred, tf.int32))
toRun = [self.output, num_correct] + grabvars
results = sess.run(toRun, feed_dict=feeder)
res = results[1]
