def build_insight_images(self):
if self.insights_x is None:
print(
"Insights requested, but no 'insights_x' in create_keras_callback() given."
)
images = []
input_data_x_sample = []
if self.has_multiple_inputs():
if not isinstance(self.insights_x, dict) and not isinstance(
self.insights_x, (list, tuple)):
raise Exception('insights_x must be a list or dict')
for i, layer in enumerate(self.model.input_layers):
x = self.insights_x[i] if isinstance(
self.insights_x, list) else self.insights_x[layer.name]
input_data_x_sample.append([x])
else:
x = self.insights_x
input_data_x_sample.append([x])
for i, layer in enumerate(self.model.input_layers):
x = input_data_x_sample[i][0]
if self.is_image_shape(x):
image = self.make_image(x)
if image:
images.append(JobImage(layer.name, image))
uses_learning_phase = self.model.uses_learning_phase
inputs = self.model.inputs[:]
if uses_learning_phase:
inputs += [K.learning_phase()]
input_data_x_sample += [0.] # disable learning_phase
layers = self.model.layers + self.insight_layer
for layer in layers:
if isinstance(
layer,
keras.layers.convolutional.Convolution2D) or isinstance(
layer, keras.layers.convolutional.MaxPooling2D):
fn = K.function(inputs, self.get_layout_output_tensors(layer))
Y = fn(input_data_x_sample)[0]
data = Y[0]
if len(data.shape) == 3:
if K.image_dim_ordering() == 'tf':
data = np.transpose(data, (2, 0, 1))
image = PIL.Image.fromarray(get_image_tales(data))
images.append(JobImage(layer.name, image))
if layer.get_weights():
data = layer.get_weights()[0]
# Keras 1 has channel only in last element when dim_ordering=tf
is_weights_channel_last = keras.__version__[
0] == '1' and K.image_dim_ordering() == 'tf'
# Keras > 1 has channel always in last element
if keras.__version__[0] != '1':
is_weights_channel_last = True
# move channel/filters to first elements to generate correct image
if is_weights_channel_last:
data = np.transpose(data, (2, 3, 0, 1))
data = data.reshape((data.shape[0] * data.shape[1],
data.shape[2], data.shape[3]))
image = PIL.Image.fromarray(get_image_tales(data))
images.append(
JobImage(layer.name + '_weights', image,
layer.name + ' weights'))
elif isinstance(layer, keras.layers.ZeroPadding2D) or isinstance(
layer, keras.layers.ZeroPadding1D) or isinstance(
layer, keras.layers.ZeroPadding3D):
pass
elif isinstance(layer,
keras.layers.noise.GaussianDropout) or isinstance(
layer, keras.layers.noise.GaussianNoise):
pass
elif isinstance(layer, keras.layers.Dropout):
pass
else:
outputs = self.get_layout_output_tensors(layer)
if len(outputs) > 0:
fn = K.function(inputs, outputs)
Y = fn(input_data_x_sample)[0]
Y = np.squeeze(Y)
if Y.size == 1:
Y = np.array([Y])
image = None
if len(Y.shape) > 1:
if len(Y.shape) == 3 and isinstance(
layer, keras.layers.Activation
) and K.image_dim_ordering() == 'tf':
Y = np.transpose(Y, (2, 0, 1))
image = PIL.Image.fromarray(get_layer_vis_square(Y))
elif len(Y.shape) == 1:
image = self.make_image_from_dense(Y)
if image:
images.append(JobImage(layer.name, image))
return images
def build_insight_images(self):
if self.insights_x is None:
self.insights_x = self.get_first_input_sample()
images = []
try:
for i, layer in enumerate(self.model.input_layers):
x = np.squeeze(self.insights_x[i])
if self.is_image_shape(x):
image = self.make_image(x)
images.append(JobImage(layer.name, image))
except:
pass
uses_learning_phase = self.model.uses_learning_phase
inputs = self.model.inputs[:]
input_data_x_sample = self.insights_x[:]
if uses_learning_phase:
inputs += [K.learning_phase()]
input_data_x_sample += [0.] # disable learning_phase
for layer in self.model.layers:
if isinstance(
layer,
keras.layers.convolutional.Convolution2D) or isinstance(
layer, keras.layers.convolutional.MaxPooling2D):
fn = K.function(inputs, self.get_layout_output_tensors(layer))
Y = fn(input_data_x_sample)[0]
data = np.squeeze(Y)
image = PIL.Image.fromarray(get_layer_vis_square(data))
images.append(JobImage(layer.name, image))
if hasattr(layer, 'W') and layer.W:
data = layer.get_weights()[0]
image = PIL.Image.fromarray(get_layer_vis_square(data))
images.append(
JobImage(layer.name + '_weights', image,
layer.name + ' weights'))
if isinstance(layer, keras.layers.Dense):
fn = K.function(inputs, self.get_layout_output_tensors(layer))
Y = fn(input_data_x_sample)[0]
Y = np.squeeze(Y)
if Y.size == 1:
Y = np.array([Y])
node = self.job_model.get_model_node(layer.name)
if node and node['activationFunction'] == 'softmax':
image = self.make_image_from_dense_softmax(Y)
else:
image = self.make_image_from_dense(Y)
images.append(JobImage(layer.name, image))
return images
def build_insight_images(self):
if self.insights_x is None:
print("Insights requested, but no 'insights_x' in create_keras_callback() given.")
images = []
input_data_x_sample = []
if self.has_multiple_inputs():
if not isinstance(self.insights_x, dict) and not isinstance(self.insights_x, (list, tuple)):
raise Exception('insights_x must be a list or dict')
for i, layer in enumerate(self.model.input_layers):
x = self.insights_x[i] if isinstance(self.insights_x, list) else self.insights_x[layer.name]
input_data_x_sample.append([x])
else:
x = self.insights_x
input_data_x_sample.append([x])
for i, layer in enumerate(self.model.input_layers):
x = input_data_x_sample[i][0]
if len(x.shape) == 3 and self.is_image_shape(x):
if K.image_dim_ordering() == 'tf':
x = np.transpose(x, (2, 0, 1))
image = self.make_image(x)
if image:
images.append(JobImage(layer.name, image))
uses_learning_phase = self.model.uses_learning_phase
inputs = self.model.inputs[:]
if uses_learning_phase:
inputs += [K.learning_phase()]
input_data_x_sample += [0.] # disable learning_phase
layers = self.model.layers + self.insight_layer
pos = 0
for layer in layers:
if isinstance(layer, keras.layers.convolutional.Convolution2D) or isinstance(layer, keras.layers.convolutional.MaxPooling2D)\
or isinstance(layer, keras.layers.convolutional.UpSampling2D):
fn = K.function(inputs, self.get_layout_output_tensors(layer))
result = fn(input_data_x_sample)
Y = result[0]
data = Y[0]
if len(data.shape) == 3:
if K.image_dim_ordering() == 'tf':
data = np.transpose(data, (2, 0, 1))
image = PIL.Image.fromarray(get_image_tales(data))
pos += 1
images.append(JobImage(layer.name, image, pos=pos))
if layer.get_weights():
data = layer.get_weights()[0]
# Keras 1 has channel only in last element when dim_ordering=tf
is_weights_channel_last = keras.__version__[0] == '1' and K.image_dim_ordering() == 'tf'
# Keras > 1 has channel always in last element
if keras.__version__[0] != '1':
is_weights_channel_last = True
# move channel/filters to first elements to generate correct image
if is_weights_channel_last:
data = np.transpose(data, (2, 3, 0, 1))
data = data.reshape((data.shape[0] * data.shape[1], data.shape[2], data.shape[3]))
image = PIL.Image.fromarray(get_image_tales(data))
pos += 1
images.append(JobImage(layer.name + '_weights', image, layer.name + ' weights', pos=pos))
elif isinstance(layer, keras.layers.ZeroPadding2D) or isinstance(layer, keras.layers.ZeroPadding1D) or isinstance(layer, keras.layers.ZeroPadding3D):
pass
elif isinstance(layer, keras.layers.noise.GaussianDropout) or isinstance(layer, keras.layers.noise.GaussianNoise):
pass
elif isinstance(layer, keras.layers.Dropout):
pass
else:
outputs = self.get_layout_output_tensors(layer)
if len(outputs) > 0:
fn = K.function(inputs, outputs)
Y = fn(input_data_x_sample)[0]
Y = np.squeeze(Y)
if Y.size == 1:
Y = np.array([Y])
image = None
if len(Y.shape) > 1:
if len(Y.shape) == 3 and self.is_image_shape(Y) and K.image_dim_ordering() == 'tf':
Y = np.transpose(Y, (2, 0, 1))
image = PIL.Image.fromarray(get_layer_vis_square(Y))
elif len(Y.shape) == 1:
image = self.make_image_from_dense(Y)
if image:
pos += 1
images.append(JobImage(layer.name, image, pos=pos))
return images
def build_insight_images(self):
if self.insights_x is None:
self.insights_x = self.get_first_input_sample()
images = []
for i, layer in enumerate(self.model.input_layers):
x = np.squeeze(self.insights_x[i])
if self.is_image_shape(x):
image = self.make_image(x)
if image:
images.append(JobImage(layer.name, image))
uses_learning_phase = self.model.uses_learning_phase
inputs = self.model.inputs[:]
input_data_x_sample = self.insights_x[:]
if uses_learning_phase:
inputs += [K.learning_phase()]
input_data_x_sample += [0.] # disable learning_phase
layers = self.model.layers + self.insight_layer
for layer in layers:
if isinstance(
layer,
keras.layers.convolutional.Convolution2D) or isinstance(
layer, keras.layers.convolutional.MaxPooling2D):
fn = K.function(inputs, self.get_layout_output_tensors(layer))
Y = fn(input_data_x_sample)[0]
data = np.squeeze(Y)
if image_data_format() == 'channels_last':
data = np.transpose(data, (2, 0, 1))
image = PIL.Image.fromarray(get_image_tales(data))
images.append(JobImage(layer.name, image))
if layer.get_weights():
data = layer.get_weights()[0]
data = np.transpose(data, (2, 3, 0, 1))
data = data.reshape((data.shape[0] * data.shape[1],
data.shape[2], data.shape[3]))
image = PIL.Image.fromarray(get_image_tales(data))
images.append(
JobImage(layer.name + '_weights', image,
layer.name + ' weights'))
elif isinstance(layer, keras.layers.ZeroPadding2D) or isinstance(
layer, keras.layers.ZeroPadding1D) or isinstance(
layer, keras.layers.ZeroPadding3D):
pass
elif isinstance(layer,
keras.layers.noise.GaussianDropout) or isinstance(
layer, keras.layers.noise.GaussianNoise):
pass
elif isinstance(layer, keras.layers.Dropout):
pass
else:
outputs = self.get_layout_output_tensors(layer)
if len(outputs) > 0:
fn = K.function(inputs, outputs)
Y = fn(input_data_x_sample)[0]
Y = np.squeeze(Y)
if Y.size == 1:
Y = np.array([Y])
node = self.job_model.get_model_node(layer.name)
image = None
if len(Y.shape) > 1:
image = PIL.Image.fromarray(get_layer_vis_square(Y))
elif len(Y.shape) == 1:
if node and node['activationFunction'] == 'softmax':
image = self.make_image_from_dense_softmax(Y)
else:
image = self.make_image_from_dense(Y)
if image:
images.append(JobImage(layer.name, image))
return images
def on_epoch_end(self, epoch, logs={}):
log = logs.copy()
self.filterInvalidJsonValues(log)
log['created'] = time.time()
log['epoch'] = epoch+1
log['loss'] = sum(self.validation_per_batch) / float(len(self.validation_per_batch))
self.validation_per_batch = []
log['validation_accuracy'] = {}
log['validation_loss'] = {}
log['training_loss'] = {}
log['training_accuracy'] = {}
trainer = self.trainer
elapsed = time.time() - self.start_time
total_loss = 0
total_accuracy = 0
for layer in trainer.model.outputs:
#todo, this is not very generic
log['validation_loss'][layer.name] = log['val_loss'] #outs[0]
log['validation_accuracy'][layer.name] = log['val_acc'] #outs[1]
log['training_loss'][layer.name] = log['loss'] #outs[0]
log['training_accuracy'][layer.name] = log['acc'] #outs[1]
total_loss += log['val_loss']
total_accuracy += log['val_acc']
if total_accuracy > self.best_total_accuracy:
self.best_total_accuracy = total_accuracy
self.best_epoch = log['epoch']
self.model.save_weights(self.filepath_best, overwrite=True)
self.model.save_weights(self.filepath_latest, overwrite=True)
if total_accuracy < self.worst_total_accuracy:
self.worst_total_accuracy = total_accuracy
self.current['totalValidationLoss'] = total_loss
self.current['totalValidationAccuracy'] = total_accuracy
self.current['totalValidationAccuracyBest'] = self.best_total_accuracy
self.current['totalValidationAccuracyWorst'] = self.worst_total_accuracy
self.current['totalValidationAccuracyBestEpoch'] = self.best_epoch
self.current['totalTrainingLoss'] = log['loss']
self.current['elapsed'] = elapsed
self.current['epoch'] = log['epoch']
self.filterInvalidJsonValues(self.current)
trainer.set_job_info('current', self.current)
line = "Epoch %d: loss=%f, acc=%f, val_loss=%f, val_acc=%f\n" % (log['epoch'], log['loss'], log.get('acc'), log['val_loss'], log.get('val_acc'), )
self.general_logger.write(line)
self.backend.job_add_status('epoch', log)
input_name = None
if self.job_model.job['insights']:
#Todo, support multiple inputs
first_input = self.model.inputs[0]
first_input_layer = self.model.input_layers[0]
first_output_layer = self.model.output_layers[0]
if first_input_layer != None:
input_data_x = self.trainer.data_validation['x'][first_input_layer.name]
input_data_y = self.trainer.data_validation['y'][first_output_layer.name]
confusion_matrix = {}
input_node = self.job_model.get_model_node(first_input_layer.name)
if self.insight_sample_input_item is None:
if self.insights_sample_path:
self.insight_sample_input_item = self.job_model.convert_file_to_input_node(self.insights_sample_path, input_node)
else:
if self.trainer.is_generator(input_data_x):
batch_x, batch_y = input_data_x.next()
self.insight_sample_input_item = batch_x[0]
else:
self.insight_sample_input_item = input_data_x[0]
# build confusion matrix
node = self.job_model.get_model_node(first_output_layer.name)
if node and node['classificationMode'] == 'categorical':
matrix = np.zeros((first_output_layer.output_shape[1], first_output_layer.output_shape[1]))
if self.trainer.is_generator(input_data_x):
processed_samples = 0
while processed_samples < trainer.nb_val_samples:
generator_output = input_data_x.next()
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
self.model._stop.set()
raise Exception('output of generator should be a tuple '
'(x, y, sample_weight) '
'or (x, y). Found: ' + str(generator_output))
if type(x) is list:
nb_samples = len(x[0])
elif type(x) is dict:
nb_samples = len(list(x.values())[0])
else:
nb_samples = len(x)
processed_samples += nb_samples
prediction = trainer.model.predict_on_batch(x)
predicted_classes = prediction.argmax(axis=-1)
expected_classes = y.argmax(axis=-1)
try:
for sample_idx, predicted_class in enumerate(predicted_classes):
expected_class = expected_classes[sample_idx]
matrix[expected_class, predicted_class] += 1
except:
pass
else:
prediction = trainer.model.predict(input_data_x, batch_size=self.job_model.get_batch_size())
predicted_classes = prediction.argmax(axis=-1)
expected_classes = input_data_y.argmax(axis=-1)
try:
for sample_idx, predicted_class in enumerate(predicted_classes):
expected_class = expected_classes[sample_idx]
matrix[expected_class, predicted_class] += 1
except:
pass
confusion_matrix[first_output_layer.name] = matrix.tolist()
images = []
try:
image = self.make_image(self.insight_sample_input_item)
images.append({
'id': input_name,
'title': input_name,
'image': self.to_base64(image)
})
except:
pass
uses_learning_phase = self.model.uses_learning_phase
inputs = [first_input]
input_data_x_sample = [[self.insight_sample_input_item]]
if uses_learning_phase:
inputs += [K.learning_phase()]
input_data_x_sample += [0.] # disable learning_phase
for layer in self.model.layers:
if isinstance(layer, keras.layers.convolutional.Convolution2D) or isinstance(layer, keras.layers.convolutional.MaxPooling2D):
fn = K.function(inputs, [layer.output])
Y = fn(input_data_x_sample)[0]
data = np.squeeze(Y)
# print("Layer Activations " + layer.name)
image = PIL.Image.fromarray(get_layer_vis_square(data))
images.append({
'id': layer.name,
'type': 'convolution',
'title': layer.name,
'image': self.to_base64(image)
})
if hasattr(layer, 'W') and layer.W:
# print("Layer Weights " + layer.name)
data = layer.W.get_value()
image = PIL.Image.fromarray(get_layer_vis_square(data))
images.append({
'id': layer.name+'_weights',
'type': 'convolution',
'title': layer.name + ' weights',
'image': self.to_base64(image)
})
if isinstance(layer, keras.layers.Dense):
fn = K.function(inputs, [layer.output])
Y = fn(input_data_x_sample)[0]
node = self.job_model.get_model_node(layer.name)
if node and node['activationFunction'] == 'softmax':
image = self.make_image_from_dense_softmax(np.squeeze(Y))
else:
image = self.make_image_from_dense(np.squeeze(Y))
images.append({
'id': layer.name,
'type': 'dense',
'title': layer.name,
'image': self.to_base64(image)
})
self.backend.job_add_insight({'epoch': log['epoch'], 'confusionMatrix': confusion_matrix}, images)