def do_testing(self, sess, cases, msg="Testing", bestk=None): inputs = [case[0] for case in cases] targets = [case[1] for case in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.error if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=sess, feed_dict=feeder, display_interval=None, testing=True) if bestk is None: print('%s Set Error = %f ' % (msg, testres)) print('%s Set Correct Classifications = %f %%' % (msg, self.gethits(cases, sess))) else: print('%s Set Correct Classifications = %f %%' % (msg, 100 * (testres / len(cases)))) return testres
def create_dendrogram(self, dendrogram_layers, number_of_cases, bestk): self.reopen_current_session() test_cases = self.caseman.get_testing_cases() cases = test_cases[:number_of_cases] inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} grabvar_names = [grabvar for grabvar in self.grabvars] grabvar_layers = [] for layer in dendrogram_layers: for grabvar in grabvar_names: if ('-'+str(layer)+'-out') in str(grabvar.name): grabvar_layers.append(grabvar) for grabvar in grabvar_layers: features = [] labels = [] for index, case in enumerate(cases): self.test_func = self.predictor if bestk is not None: self.test_func = self.gen_match_counter(self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, grabvar, self.probes, session=self.current_session, feed_dict=feeder, show_interval=self.show_interval, step=self.global_training_step) labels.append(TFT.bits_to_str(case[1])) features.append(grabvals[index]) TFT.dendrogram(features, labels, title=grabvar.name)
def consider_early_stopping(self, sess, cases, msg="Early Stopping", bestk=None, target_accuracy=None): inputs = [case[0] for case in cases] targets = [case[1] for case in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.error if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=sess, feed_dict=feeder, display_interval=None, testing=True) print("\n CONSIDER EARLY STOPPING: \n") if bestk is None: print('%s Set Correct Classifications = %f %% \n' % (msg, self.gethits(cases, sess))) if self.gethits(cases, sess) > target_accuracy: return True else: print('%s Set Correct Classifications = %f %% \n' % (msg, 100 * (testres / len(cases)))) if 100 * (testres / len(cases)) > target_accuracy: return True print("\n Target Accuracy NOT reached - continue: \n") return False
def do_mapping(self, no_of_cases, defined_grab_vars=[], dendrogram_layers=[], bestk=None, dendrogram=False, labels=False): self.reopen_current_session() if len(defined_grab_vars) > 0: for (layer, type) in defined_grab_vars: if (layer-1, 'in') not in defined_grab_vars: self.add_grabvar(layer-1, type) if len(dendrogram_layers) > 0: for layer in dendrogram_layers: self.add_dendrogram_var(layer-1, 'out') cases = self.get_random_cases(no_of_cases) inputs = [c[0] for c in cases] targets = [c[1] for c in cases] if bestk is not None: self.test_func = self.gen_match_counter(self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) feeder = {self.input: inputs, self.target: targets} testres, grabvals, _ = self.run_one_step(self.test_func, [self.grabvars, self.dendrogram_vars], self.probes, session=self.current_session, feed_dict=feeder) self.display_grabvars(grabvals[0], self.grabvars) if dendrogram: if labels: input_den = [target for target in targets] else: input_den = [TFT.bits_to_str(case) for case in inputs] for i, grabval in enumerate(grabvals[1]): TFT.dendrogram(grabval, input_den, title=self.dendrogram_vars[i].name)
def do_mapping(self, map_layers, number_of_cases, bestk): self.reopen_current_session() test_cases = self.caseman.get_training_cases() cases = test_cases[:number_of_cases] inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} grabvar_names = [grabvar for grabvar in self.grabvars] grabvar_layers = [] for layer in map_layers: for grabvar in grabvar_names: if ('-'+str(layer)+'-out') in str(grabvar.name): grabvar_layers.append(grabvar) for grabvar in grabvar_layers: features = [] for index, case in enumerate(cases): self.test_func = self.predictor if bestk is not None: self.test_func = self.gen_match_counter(self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, grabvar, self.probes, session=self.current_session, feed_dict=feeder, show_interval=self.show_interval, step=self.global_training_step) features.append(grabvals[index]) TFT.hinton_plot(np.array(features), fig=PLT.figure(), title=grabvar.name + " Activation Levels") new_target = np.array(targets) TFT.hinton_plot(new_target, fig=PLT.figure(), title="Input Targets")
def do_testing(self, sess, cases, msg='Testing', bestk=None): inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.error if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=sess, feed_dict=feeder, show_interval=None) if bestk is None: print('%s Set Error = %f ' % (msg, testres)) else: print('%s Set Correct Classifications = %f %%' % (msg, 100 * (testres / len(cases)))) return testres # self.error uses MSE, so this is a per-case value when bestk=None
def do_mapping(self, sess, cases, start=0, msg='Mapping', bestk=None): inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.predictor if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) results_hinton = sess.run([self.test_func, self.grabvars[0:start]], feed_dict=feeder) dendro_cases = self.caseman.get_training_cases() dendro_inputs = [c[0] for c in dendro_cases] dendro_targets = [c[1] for c in dendro_cases] dendro_feeder = { self.input: dendro_inputs, self.target: dendro_targets } results_dendro = sess.run([self.test_func, self.grabvars[start:]], feed_dict=dendro_feeder) #print(dendro_dict) self.display_grabvars(results_hinton[1], self.grabvars[0:start], step=1) for t in targets: print("True Label: ", t) total_labels = [] total_targets = [] for j in range(1, len(results_dendro)): dendro_dict = {} for i in range(len(results_dendro[j])): for k in range(len(results_dendro[j][i])): inp = dendro_targets[k] res = results_dendro[j][i][k] if (type(inp) == int): string = str(inp) else: string = TFT.bits_to_str(inp) if (string not in dendro_dict): dendro_dict[string] = [i for i in res] dendro_labels = [] dendro_data = [] for key in dendro_dict.keys(): dendro_labels.append(key) dendro_data.append(dendro_dict[key]) #print("Labels: ", dendro_labels) #print("Targets: ", dendro_targets) total_labels.append(dendro_labels) total_targets.append(dendro_data) self.display_dendrograms(total_targets, self.grabvars[start:], total_labels, start)
def do_mapping(self, session, cases): inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} testFunc = self.gen_match_counter(self.output, [TFT.one_hot_to_int(list(v)) for v in targets], k=1) _, vals, _ = self.run_one_step(grabbed_vars = [self.mapVars] + [self.dendrogramVars] + [self.grabVars], probed_vars = self.probes, operators = testFunc, session = session, feed_dict = feeder, showInterval = 0) mapped_vals = vals[0] dendrogram_vals = vals[1] grabbed_vals = vals[2] return mapped_vals, dendrogram_vals, grabbed_vals
def do_mapping(self, sess, cases, msg='Mapping', bestk=1): inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.error if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) error, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=sess, feed_dict=feeder, show_interval=None) return grabvals # self.error uses MSE, so this is a per-case value
def do_testing(self, sess, cases, epoch = 'test', msg='Testing', bestk=1): inputs = [c[0] for c in cases]; targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} testFunc = self.gen_match_counter(self.output, [TFT.one_hot_to_int(list(v)) for v in targets], k = 1) accuracy, _, _ = self.run_one_step(grabbed_vars = self.grabVars, probed_vars = self.probes, operators = testFunc, session = sess, feed_dict = feeder, showInterval = 0) loss, _, _ = self.run_one_step(grabbed_vars=self.grabVars, probed_vars=self.probes, operators=self.error, session=sess, feed_dict=feeder, showInterval=0) accuracy = 100 * (accuracy / len(cases)) if bestk is None: print('Epoch: %s\t\t %s loss: N/A\t\t %s : %s %%' % (epoch, msg, msg, accuracy)) else: print('Epoch: %s\t\t %s loss: %5.4f\t\t %s accuracy: %d %%' % (epoch, msg, loss, msg, accuracy)) return loss
def run_mapping(self, case_generator = None, mapBatchSize = 0, mapLayers = [], mapDendrograms = []): self.mapBatchSize = mapBatchSize # Size of batch of cases used for a map test. 0 indicates no map test self.mapLayers = mapLayers # List of layers(their indices) to be visualized during a map test self.mapDendrograms = mapDendrograms # List of layers(their indices) whose activation patterns will be used to make dendrograms self.reopen_current_session() if self.mapBatchSize: # either a chosen set of cases or a random subset of the training cases: cases = case_generator() if case_generator else self.caseMan.get_mapping_cases(self.mapBatchSize) # Add the monitored variables self.add_mapvars() self.add_dendrogramvars() # Add all grabbed variables for weight in self.displayWeights: self.add_grabvar(weight, 'wgt') for bias in self.displayBiases: self.add_grabvar(bias, 'bias') # run map test mapvals, dendrovals, grabvals = self.do_mapping(session = self.current_session, cases = cases) # Plotting names = [x.name for x in self.mapVars] for i, v in enumerate(mapvals): if type(v) == np.ndarray and len(v.shape) > 1: # If v is a matrix, use hinton plotting TFT.hinton_plot(v, fig=None, title='Activation pattern of layer ' + names[i]) if len(self.mapDendrograms) > 0: names = [x.name for x in self.dendrogramVars] if TFT.is_bit_vector(cases[0][0]): labels = [TFT.bits_to_str(s[0]) for s in cases] else: labels = [TFT.one_hot_to_int(c[1]) for c in cases] for (i, v) in enumerate(dendrovals): TFT.dendrogram(v, labels, title = 'Dendrogram of ' + names[i]) if len(grabvals) > 0: self.display_grabvars(grabvals, self.grabVars, step = self.globalTrainingStep) # hold until a button is pressed while (not PLT.waitforbuttonpress()): pass PLT.close('all')
def do_mapping(self, msg="Mapping", bestk=None, plot=""): self.reopen_current_session() cases = self.caseman.getMappingCases(self.mapBatchSize) self.grabvars.clear() helpers.add_grabvars(self, self.wantedMapGrabvars) helpers.add_grabvars(self, self.dendrogramLayers) inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.predictor if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=self.current_session, feed_dict=feeder, show_interval=None) if (self.mapplot == 'display'): eval("TFT.display_matrix(testres)") elif (self.mapplot == "hinton"): eval("TFT.hinton_plot(testres)") elif (plot == "dendrogram"): eval("TFT.dendrogram(inputs, targets)") # Dendrogram for each layer... # print("grabvals") # print(grabvals) for hiddenactivation in grabvals[len(self.wantedMapGrabvars):]: # index, type TFT.dendrogram(hiddenactivation, targets, ax=None) # TFT.dendrogram(feature, labels) --> mappe input og hidden activation patterns ... waht are the input, and what is the labels ? I see that labels are suppose to be string.. # TFT.dendrogram(testres, targets, ax=None) # ser ut til å funke, men er jo feile verdier ? self.close_current_session(view=False)
def do_mapping(self, sess, cases, msg='Mapping', bestk=None): # must reopen current session to work inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} self.test_func = self.predictor if bestk is not None: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=sess, feed_dict=feeder, show_interval=1) print(np.shape(testres)) print(np.shape(grabvals))
def do_testing(self, sess, cases, msg='Testing', bestk=None, epoch=0): inputs = [c[0] for c in cases] targets = [c[1] for c in cases] feeder = {self.input: inputs, self.target: targets} #print("\n--- " + msg + " ---\n") self.test_func = self.error if bestk is not None: if (self.loss_func == "sparse_softmax_cross_entropy"): self.test_func = self.gen_match_counter(self.predictor, [v for v in targets], k=bestk) else: self.test_func = self.gen_match_counter( self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets], k=bestk) testres, grabvals, _ = self.run_one_step(self.test_func, self.grabvars, self.probes, session=sess, feed_dict=feeder, show_interval=None) if bestk is None: if epoch == 0: epoch = self.global_training_step if (type(testres) != np.float64 and len(testres) > 1): batch_error = 0 for e in testres: batch_error += e batch_error = float(batch_error / len(testres)) print('%s Set Error = %f ' % (msg, batch_error)) else: print('%s Set Error = %f ' % (msg, testres)) else: print('%s Set Correct Classifications = %f %%' % (msg, 100 * (testres / len(cases)))) return testres # self.error uses MSE, so this is a per-case value when bestk=None
error = self.do_testing(case_list, scenario="validation") # print(step, ", error=", error) self.validation_error_history.append((step, error)) def do_testing(self, case_list=None, scenario="testing", grabvars=[]): if not grabvars: grabvars = self.grabvars if scenario == "testing" and not case_list: case_list = self.case_manager.get_testing_cases() # print(case_list[0].input) # print(case_list[0].target) inputs = [case.input for case in case_list] targets = [case.target for case in case_list] pred_targets = [t[0] if len(t) == 1 else TFT.one_hot_to_int(t) for t in targets] sess = self.sess feeder = {self.input: inputs, self.target: targets} if self.output_layer_size == 1: # one outout node, special case correct_pred = tf.equal(tf.round(self.output), targets) num_correct = tf.reduce_sum(tf.cast(correct_pred, tf.int32)) else: correct_pred = tf.nn.in_top_k(tf.cast(self.output, tf.float32), tf.cast(pred_targets, tf.int32), self.config.topk) num_correct = tf.reduce_sum(tf.cast(correct_pred, tf.int32)) toRun = [self.output, num_correct] + grabvars results = sess.run(toRun, feed_dict=feeder) res = results[1]