def create_dendrogram(self, dendrogram_layers, number_of_cases, bestk):
self.reopen_current_session()
test_cases = self.caseman.get_testing_cases()
cases = test_cases[:number_of_cases]
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
grabvar_names = [grabvar for grabvar in self.grabvars]
grabvar_layers = []
for layer in dendrogram_layers:
for grabvar in grabvar_names:
if ('-'+str(layer)+'-out') in str(grabvar.name):
grabvar_layers.append(grabvar)
for grabvar in grabvar_layers:
features = []
labels = []
for index, case in enumerate(cases):
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(self.predictor,
[TFT.one_hot_to_int(list(v)) for v in targets], k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func, grabvar, self.probes,
session=self.current_session,
feed_dict=feeder, show_interval=self.show_interval,
step=self.global_training_step)
labels.append(TFT.bits_to_str(case[1]))
features.append(grabvals[index])
TFT.dendrogram(features, labels, title=grabvar.name)
def gen_dendrogram(self, numCases, msg="dendogram"):
names = [x.name for x in self.grabvars]
self.reopen_current_session()
tCases = self.case_manager.get_training_cases()
for j in range(0, len(self.grabvars)):
features = []
labels = []
for i in range(0, numCases):
print(i)
case = tCases[i]
feeder = {self.input: [case[0]], self.target: [case[1]]}
result = self.current_session.run([self.output, self.grabvars],
feed_dict=feeder)
r = result[1][j][0]
print("\n\n")
print(result[1][j][0])
print("\n\n")
features.append(r)
labels.append(TFT.bits_to_str(case[0]))
print(features)
print(labels)
#print(features)
#print(labels)
name = str(names[j]) + "Dendrogram"
TFT.dendrogram(features, labels, title=name)
def do_mapping(self, no_of_cases, defined_grab_vars=[], dendrogram_layers=[], bestk=None, dendrogram=False, labels=False):
self.reopen_current_session()
if len(defined_grab_vars) > 0:
for (layer, type) in defined_grab_vars:
if (layer-1, 'in') not in defined_grab_vars:
self.add_grabvar(layer-1, type)
if len(dendrogram_layers) > 0:
for layer in dendrogram_layers:
self.add_dendrogram_var(layer-1, 'out')
cases = self.get_random_cases(no_of_cases)
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
if bestk is not None:
self.test_func = self.gen_match_counter(self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
feeder = {self.input: inputs, self.target: targets}
testres, grabvals, _ = self.run_one_step(self.test_func, [self.grabvars, self.dendrogram_vars], self.probes, session=self.current_session,
feed_dict=feeder)
self.display_grabvars(grabvals[0], self.grabvars)
if dendrogram:
if labels:
input_den = [target for target in targets]
else:
input_den = [TFT.bits_to_str(case) for case in inputs]
for i, grabval in enumerate(grabvals[1]):
TFT.dendrogram(grabval, input_den, title=self.dendrogram_vars[i].name)
def do_mapping(self,
session=None,
scatter=True,
mbs=100,
testset="mapping",
mapbs=10,
numeric=False):
sess = session if session else self.current_session
grabvars = [self.input, self.predictor]
grabvars += self.mapvars
randNum = random.randint(0, len(self.training_cases) - mapbs)
cases = self.training_cases[randNum:randNum + mapbs]
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
predictions = []
feeder = {self.input: inputs, self.target: targets}
_, grabvals, _ = self.run_one_step([self.predictor],
grabvars,
session=sess,
feed_dict=feeder,
show_interval=None)
for val in grabvals[1]:
predictions.append(val)
fig_index = 0
names = [x.name for x in grabvars[2:-len(self.dendrogram_layers)]]
for grabval in grabvals[2:-len(self.dendrogram_layers)]:
if (type(grabval[0]) != np.ndarray):
grabval = np.array([[c] for c in grabval])
if numeric:
TFT.display_matrix(grabval,
fig=self.mapvar_figures[fig_index],
title=names[fig_index])
else:
TFT.hinton_plot(grabval,
fig=self.mapvar_figures[fig_index],
title=names[fig_index])
fig_index += 1
input_strings = [TFT.bits_to_str(i) for i in inputs]
target_strings = [TFT.bits_to_str(i) for i in targets]
for dendro_vals in grabvals[-len(self.dendrogram_layers):]:
TFT.dendrogram(dendro_vals, target_strings)
a = input()
def display_dendrograms(self,
grabbed_vals,
grabbed_vars,
labels,
start,
title="Dendrogram",
step=0):
names = [x.name for x in grabbed_vars]
msg = "Grabbed Variables at Step " + str(step)
#print("\n" + msg, end="\n")
fig_index = start
for i, v in enumerate(grabbed_vals):
#print(v, names[i])
fig = self.grabvar_figures[fig_index]
if fig == None:
print('FIGURE IS NONE')
TFT.dendrogram(v, labels[i], title=self.title + "_" + names[i])
fig_index += 1
def run_mapping(self, case_generator = None, mapBatchSize = 0, mapLayers = [], mapDendrograms = []):
self.mapBatchSize = mapBatchSize # Size of batch of cases used for a map test. 0 indicates no map test
self.mapLayers = mapLayers # List of layers(their indices) to be visualized during a map test
self.mapDendrograms = mapDendrograms # List of layers(their indices) whose activation patterns will be used to make dendrograms
self.reopen_current_session()
if self.mapBatchSize:
# either a chosen set of cases or a random subset of the training cases:
cases = case_generator() if case_generator else self.caseMan.get_mapping_cases(self.mapBatchSize)
# Add the monitored variables
self.add_mapvars()
self.add_dendrogramvars()
# Add all grabbed variables
for weight in self.displayWeights:
self.add_grabvar(weight, 'wgt')
for bias in self.displayBiases:
self.add_grabvar(bias, 'bias')
# run map test
mapvals, dendrovals, grabvals = self.do_mapping(session = self.current_session, cases = cases)
# Plotting
names = [x.name for x in self.mapVars]
for i, v in enumerate(mapvals):
if type(v) == np.ndarray and len(v.shape) > 1: # If v is a matrix, use hinton plotting
TFT.hinton_plot(v, fig=None, title='Activation pattern of layer ' + names[i])
if len(self.mapDendrograms) > 0:
names = [x.name for x in self.dendrogramVars]
if TFT.is_bit_vector(cases[0][0]):
labels = [TFT.bits_to_str(s[0]) for s in cases]
else:
labels = [TFT.one_hot_to_int(c[1]) for c in cases]
for (i, v) in enumerate(dendrovals):
TFT.dendrogram(v, labels, title = 'Dendrogram of ' + names[i])
if len(grabvals) > 0:
self.display_grabvars(grabvals, self.grabVars, step = self.globalTrainingStep)
# hold until a button is pressed
while (not PLT.waitforbuttonpress()):
pass
PLT.close('all')
def display_mapvars(self,
grabbed_vals,
grabbed_vars,
mode='h',
cases=None):
names = [x.name for x in grabbed_vars]
fig_index = 0
for i, v in enumerate(grabbed_vals):
if mode == 'h': # If v is a matrix
TFT.hinton_plot(v,
fig=self.mapLayer_grabvar_figures[fig_index],
title=names[i])
fig_index += 1
elif mode == 'd': # If v is a matrix
labels = []
for j in cases:
labels.append(''.join(str(e) for e in j[0]))
grabbed_vals = grabbed_vals[0].tolist()
TFT.dendrogram(grabbed_vals, labels)
def do_mapping(self, sess, cases):
#Separate the cases into inputs and targets
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
#Getting the value for the label
tar = []
for t in targets:
tar.append(list(t).index(1))
#Running the network without learning on the map batch
feeder = {self.input: inputs, self.target: targets}
testres, grabvals, _ = self.run_one_step(self.predictor,
self.grabvars,
self.probes,
session=sess,
feed_dict=feeder,
show_interval=None)
#Putting the names into an array for easier look up
names = [x.name for x in self.grabvars]
zips = zip(names, grabvals)
#Variables to show which
num = 0
num2 = 0
#Plotting the different plots
for i in zips:
if (i[0] in self.Hnames):
TFT.hinton_plot(i[1],
fig=PLT.figure(),
title=i[0] + ' at step ' + str("Test"))
if (i[0] in self.Dnames):
TFT.dendrogram(i[1], tar, title="dendrogram" + str(i[0]))
if (i[0] in self.wnames):
fig_wgt = PLT.figure()
TFT.display_matrix(i[1], fig=fig_wgt, title=(i[0]))
num += 1
if (i[0] in self.bnames):
fig_bias = PLT.figure()
TFT.display_matrix(np.array([i[1]]),
fig=fig_bias,
title=(i[0]))
num2 += 1
def do_mapping(self, msg="Mapping", bestk=None, plot=""):
self.reopen_current_session()
cases = self.caseman.getMappingCases(self.mapBatchSize)
self.grabvars.clear()
helpers.add_grabvars(self, self.wantedMapGrabvars)
helpers.add_grabvars(self, self.dendrogramLayers)
inputs = [c[0] for c in cases]
targets = [c[1] for c in cases]
feeder = {self.input: inputs, self.target: targets}
self.test_func = self.predictor
if bestk is not None:
self.test_func = self.gen_match_counter(
self.predictor, [TFT.one_hot_to_int(list(v)) for v in targets],
k=bestk)
testres, grabvals, _ = self.run_one_step(self.test_func,
self.grabvars,
self.probes,
session=self.current_session,
feed_dict=feeder,
show_interval=None)
if (self.mapplot == 'display'):
eval("TFT.display_matrix(testres)")
elif (self.mapplot == "hinton"):
eval("TFT.hinton_plot(testres)")
elif (plot == "dendrogram"):
eval("TFT.dendrogram(inputs, targets)")
# Dendrogram for each layer...
# print("grabvals")
# print(grabvals)
for hiddenactivation in grabvals[len(self.wantedMapGrabvars):]:
# index, type
TFT.dendrogram(hiddenactivation, targets, ax=None)
# TFT.dendrogram(feature, labels) --> mappe input og hidden activation patterns ... waht are the input, and what is the labels ? I see that labels are suppose to be string..
# TFT.dendrogram(testres, targets, ax=None) # ser ut til å funke, men er jo feile verdier ?
self.close_current_session(view=False)
def display_dendrograms(self,
grabbed_vals,
grabbed_vars,
step,
onezero=False,
komma=False,
punktum=False,
decimals=1,
sleep_time=30,
leaf_font_size=None):
names = [x.name for x in grabbed_vars]
msg = "Grabbed Variables at Step " + str(step)
print("\n" + msg, end="\n")
for j, i, o in zip(range(len(grabbed_vals[0:len(grabbed_vals):2])),
grabbed_vals[0:len(grabbed_vals):2],
grabbed_vals[1:len(grabbed_vals):2]):
in_pattern = []
for line in i:
in_vals = line
for element in line:
# Format long floats, if one of the numbers is not an int.
if not element.is_integer():
in_vals = []
for element in line:
# Represent '0,0' as 'o'
if element == 0 and onezero == 1:
in_vals.append("o")
# Ceil the float by with specified number of decimals
else:
deci = "%." + str(decimals) + "f"
in_vals.append(deci % element)
break
# Keep '.'
if punktum:
din = TFT.bits_to_str(in_vals, komma)
# Remove '.'
else:
din = TFT.bits_to_str(in_vals, komma).replace(".", "")
# Gather only unique cases
if not din in in_pattern:
in_pattern.append(din)
if not self.allUnique(in_pattern):
in_pattern.pop()
if TFT.dendrogram(
o,
in_pattern,
title="dendrogram: " +
names[self.displayed_dendrograms].replace('out', ''),
sleep_time=sleep_time,
leaf_font_size=leaf_font_size):
self.displayed_dendrograms += 2
def _visualize_dendrogram(self):
print('Print dendro')
if 'map_dendrograms' in self.properties.keys(
) and self.properties['map_batch_size'] > 0:
act = self.net.custom_run(
[
self.net.A[a - 1]
for a in self.properties['map_dendrograms']
], {
self.net.x:
self.data_set.training.x[0:self.
properties['map_batch_size']]
})
labels = []
for l in self.data_set.training.x[:self.
properties['map_batch_size']]:
s = ''
for b in l:
s += str(b) + '\n'
labels.append(s)
tft.dendrogram(act[0], labels)
def display_dendrogram(self, vals, inputs, title):
TFT.dendrogram(vals, inputs, title="Dendrogram: " + title)
def train(dims=[11,40,20,6],
activation_func='tanh',
softmax=True,
cost_func=CE,
lr= 0.5,
vint = 10,
bint = 10,
acc_lim = 0.95,
initial_weight_range=[-0.1,0.1],
data_source='gen_wine_cases',
case_count=1,
vfrac=0.1,
tfrac=0.1,
mbs=1277,
map_bs=20,
epochs=10000,
show_layers=None,
dendogram_layers=None,
show=True,
map_layers = [1, 2]):
#Training and validation accuracies
train_acc= []
val_acc = []
# Import data
dataset = getattr(TFT,data_source)(case_count=case_count)
mnist = case_holder(dataset,tfrac=tfrac,vfrac=vfrac)
sess = tf.InteractiveSession()
# Create a multilayer model.
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, dims[0]], name='x-input')
y_ = tf.placeholder(tf.float32, [None, dims[-1]], name='y-input')
# We can't initialize these variables to 0 - the network will get stuck.
def weight_variable(shape):
"""Create a weight variable with appropriate initialization."""
if initial_weight_range == "scaled":
initial = tf.truncated_normal(shape, stddev=0.1)
else:
initial = tf.Variable(tf.random_uniform(shape=shape,minval=initial_weight_range[0],maxval=[initial_weight_range[1]]))
return tf.Variable(initial)
def bias_variable(shape):
"""Create a bias variable with appropriate initialization."""
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def variable_summaries(var):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=getattr(tf.nn,activation_func)):
"""Reusable code for making a simple neural net layer.
It does a matrix multiply, bias add, and then uses ReLU to nonlinearize.
It also sets up name scoping so that the resultant graph is easy to read,
and adds a number of summary ops.
"""
# Adding a name scope ensures logical grouping of the layers in the graph.
with tf.name_scope(layer_name):
# This Variable will hold the state of the weights for the layer
with tf.name_scope('weights'):
weights = weight_variable([input_dim, output_dim])
variable_summaries(weights)
with tf.name_scope('biases'):
biases = bias_variable([output_dim])
variable_summaries(biases)
with tf.name_scope('Wx_plus_b'):
preactivate = tf.matmul(input_tensor, weights) + biases
tf.summary.histogram('pre_activations', preactivate)
activations = act(preactivate, name='activation')
tf.summary.histogram('activations', activations)
return activations
previous_layer = x
layers = []
for i in range(1,len(dims)):
layers.append(nn_layer(previous_layer,dims[i-1],dims[i],'layer'+str(i),act=tf.nn.relu))
previous_layer = layers[-1]
y = layers[-1]
with tf.name_scope('error_func'):
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the
# raw outputs of the nn_layer above, and then average across
# the batch.
diff = error_funcs[cost_func](y_,y)
#diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(learning_rate=lr).minimize(
cross_entropy)
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Merge all the summaries and write them out to
# /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter('netsaver_test'+ '/train', sess.graph)
test_writer = tf.summary.FileWriter('netsaver_test' + '/test')
tf.global_variables_initializer().run()
# Train the model, and also write summaries.
# Every 10th step, measure test-set accuracy, and write test summaries
# All other steps, run train_step on training data, & add training summaries
def feed_dict(train):
"""Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
if train == "train":
xs, ys = mnist.train_next_batch(size=mbs)
elif train == 'test':
xs, ys = mnist.test_features, mnist.test_labels
elif train == 'val':
xs, ys = mnist.validation_features, mnist.validation_labels
elif train == 'map':
xs, ys = mnist.train_features[:map_bs], mnist.train_labels[:map_bs]
else:
raise Exception
return {x: xs, y_: ys}
for i in range(epochs):
if i % bint == 0: # Record summaries and test-set accuracy
summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict('train'))
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
# Own code for pulling training accuracy to matplot graph
train_acc.append([i,acc])
if acc >= acc_lim: break
else: # Record train set summaries, and train
if i % 100 == 99: # Record execution stats
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict('train'),
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
print('Adding run metadata for', i)
else: # Record a summary
summary, _ = sess.run([merged, train_step], feed_dict=feed_dict('train'))
train_writer.add_summary(summary, i)
train_writer.close()
test_writer.close()
# Display final test scores
summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict('train'))
print('Final training set accuracy: %s' % ( acc))
# Code for displaying graphs
if show:
TFT.plot_training_history(train_acc,val_acc)
if map_layers:
for l in map_layers:
_, activation = sess.run([merged,layers[l]],feed_dict=feed_dict('map'))
TFT.display_matrix(activation, title="mapping of layer: "+ str(l))
# for variable in tf.trainable_variables():
# if variable.name in real-map_layer:
# _,values = sess.run([merged,variable],feed_dict=feed_dict('map'))
# if 'weigths' in variable.name:
# TFT.display_matrix(values)
# elif 'biases' in variable.name:
# TFT.display_vector(values)
# else:
# raise Exception("wrong dimensionality on show layers")
if show_layers:
for variable in tf.trainable_variables():
if variable.name in show_layers:
_,values = sess.run([merged,variable],feed_dict=feed_dict('map'))
if len(values.shape) == 2:
TFT.display_matrix(values, title="weights of: "+variable.name)
elif len(values.shape) == 1:
TFT.display_vector(values, title="biases of: "+variable.name)
else:
raise Exception("wrong dimensionality on map layers")
if dendogram_layers:
for l in dendogram_layers:
_, activation = sess.run([merged,layers[l]],feed_dict=feed_dict('map'))
y_s = []
#for y in feed_dict('map')[x]:
# y_s.append(TFT.segmented_vector_string(y))
TFT.dendrogram(activation,feed_dict('map')[y_], title="Dendogram, layer: "+str(l))
PLT.show()
#making dendrograms
if len(map_dendrograms) > len(map_layers):
raise ValueError(
'Custom error: Layers for dendrograms must be mapped via map_layers (len(map_dendrograms) > len(map_layers))'
)
saved_m_index = 0
for module_index in map_dendrograms:
all_inputs = []
all_outputs = []
for case in all_module_vals:
all_inputs.append(case[saved_m_index][0])
all_outputs.append(case[saved_m_index][1])
title = "Dendrogram module %d: output" % module_index
print("displaying dendrograms...")
TFT.dendrogram(all_outputs, None, title=title)
PLT.pause(1)
saved_m_index += 1
# **** MAIN functions ****
# After running this, open a Tensorboard (Go to localhost:6006 in your Chrome Browser) and check the
# 'scalar', 'distribution' and 'histogram' menu options to view the probed variables.
def autoex(dims=[8, 4, 8],
activation_hidden='relu',
activation_output='softmax',
cost_function='softmax_cross_entropy_with_logits',
lrate=0.25,
init_weight_range=[-.1, .1],
data_source=['glass.txt', [0]],
cfrac=1,
def main():
parser = argument_parser.argument_parser()
parser.parse()
parser.organize()
# (self, cases, vfrac, tfrac, casefrac, mapsep)
caseman = gann_base.Caseman(parser.data_set_v, parser.vfrac_v,
parser.tfrac_v, parser.casefrac_v,
parser.mapbs_v)
# (self, dims, cman, afunc, ofunc, cfunc, optimizer, lrate, wrange, vint, mbs, usevsi, showint=None):
ann = gann_base.Gann(parser.dims_v,
caseman,
parser.afunc_v,
parser.ofunc_v,
parser.cfunc_v,
parser.optimizer_v,
parser.lrate_v,
parser.wrange_v,
parser.vint_v,
parser.mbs_v,
parser.usevsi_v,
showint=parser.steps_v - 1)
for layer in parser.dispw_v:
ann.add_grabvar(layer, type='wgt')
ann.gen_probe(layer, 'wgt', 'hist')
for layer in parser.dispb_v:
ann.add_grabvar(layer, type='bias')
ann.gen_probe(layer, 'bias', 'hist')
# run, then map
ann.run(steps=parser.steps_v,
sess=None,
continued=False,
bestk=parser.best1_v)
ann.remove_grabvars()
for layer in parser.maplayers_v:
if layer == 0:
ann.add_grabvar(layer, type='in', add_figure=False)
else:
ann.add_grabvar(layer - 1, type='out', add_figure=False)
res, labs = ann.do_mapping()
results = []
for i in range(len(res[0])):
l = np.array([r[i] for r in res])
l = l.reshape(l.shape[0], l.shape[2])
TFT.hinton_plot(l,
title="mapping test output of layer " +
str(parser.maplayers_v[i]))
results.append(l)
for i, r in enumerate(results):
# DENDOGRAM
# if parser.maplayers_v[i] in parser.mapdend_v:
if parser.best1_v:
TFT.dendrogram(r,
list(map(TFT.one_hot_to_int, labs)),
title="Dendrogram " + str(parser.maplayers_v[i]))
gann_base.PLT.show()
TFT.fireup_tensorboard('probeview')