def _make_mv_model(self, filepath):
""" Create a model with moving averaged weights. Other variables are
the same as original mode. We first save original model to save its
state. Then copy moving averaged weights over."""
self.model.save(filepath, overwrite=True)
model2 = load_model(filepath, custom_objects=self.custom_objects)
for w2, w in zip(collect_trainable_weights(model2),
collect_trainable_weights(self.model)):
K.set_value(w2, self.mv_trainable_weights_vals[w.name])
return model2
def test_sequential_trainable():
x = Input(shape=(20,))
dense1 = Dense(20)
dense2 = Dense(10)
dense3 = Dense(1)
seq = sequential([
dense1,
dense2,
dense3,
], trainable=False)
seq(x)
assert collect_trainable_weights(dense1) == []
assert collect_trainable_weights(dense2) == []
assert collect_trainable_weights(dense3) == []
def test_sequential_trainable():
x = Input(shape=(20, ))
dense1 = Dense(20)
dense2 = Dense(10)
dense3 = Dense(1)
seq = sequential([
dense1,
dense2,
dense3,
], trainable=False)
seq(x)
assert collect_trainable_weights(dense1) == []
assert collect_trainable_weights(dense2) == []
assert collect_trainable_weights(dense3) == []
def _build(self):
fake, _, _, g_additional_losses = self.g.run_internal_graph(self.g.inputs)
real = self.d.inputs[0]
data = concat([fake, real], axis=0)
realness, _, _, d_additional_losses = self.d.run_internal_graph(
[data] + self.d.inputs[1:])
nb_fakes = fake.shape[0]
fake_realness = realness[:nb_fakes]
real_realness = realness[nb_fakes:]
split = 2*nb_fakes // 3
g_fake_realness = fake_realness[:split]
d_fake_realness = fake_realness[split:]
outputs = OrderedDict()
g_loss = K.mean(K.binary_crossentropy(g_fake_realness, K.ones_like(real_realness)))
outputs['g_loss'] = g_loss
g_reg_loss = sum([v for v in g_additional_losses.values()])
if g_reg_loss != 0:
outputs['g_reg_loss'] = g_reg_loss
g_total_loss = g_loss + g_reg_loss
d_loss = K.mean(K.binary_crossentropy(real_realness, K.ones_like(real_realness)))
d_loss += K.mean(K.binary_crossentropy(d_fake_realness, K.zeros_like(real_realness)))
outputs['d_loss'] = d_loss
d_reg_loss = sum([v for v in d_additional_losses.values()])
if d_reg_loss != 0:
outputs['d_reg_loss'] = d_reg_loss
d_total_loss = d_loss + d_reg_loss
inputs = {i.name: i for i in self.g.inputs + self.d.inputs}
inputs_list = []
for name in self.input_names:
inputs_list.append(inputs[name])
g_updates = self.g_optimizer.get_updates(
collect_trainable_weights(self.g), self.g.constraints, g_total_loss)
d_updates = self.d_optimizer.get_updates(
collect_trainable_weights(self.d), self.d.constraints, d_total_loss)
if self.uses_learning_phase:
lr_phase = [K.learning_phase()]
else:
lr_phase = []
self.metrics_names = list(outputs.keys())
self._train_function = K.function(inputs_list + lr_phase, list(outputs.values()),
updates=g_updates + d_updates)
def print_parameters(self):
from keras.engine.training import collect_trainable_weights
logger.info("total number of parameters: %s" %
self.graph.count_params())
trainable_weights = collect_trainable_weights(self.graph)
total = sum([K.count_params(p) for p in trainable_weights])
logger.info("number of trainable parameters: %s" % total)
def on_train_begin(self, logs={}):
self.sym_trainable_weights = collect_trainable_weights(self.model)
# Initialize moving averaged weights using original model values
self.mv_trainable_weights_vals = {
x.name: K.get_value(x)
for x in self.sym_trainable_weights
}
if self.verbose:
print('Created a copy of model weights to initialize moving'
' averaged weights.')
def print_parameters(self):
from keras.engine.training import collect_trainable_weights
logger.info("total number of parameters: %s" % self.graph.count_params())
trainable_weights = collect_trainable_weights(self.graph)
total = sum([K.count_params(p) for p in trainable_weights])
logger.info("number of trainable parameters: %s" % total)
def XXX_get_trainable_params(self, model):
params = []
for layer in model.layers:
params += training.collect_trainable_weights(layer)
return params
def XXX_get_trainable_params(self, model):
params = []
for layer in model.layers:
params += training.collect_trainable_weights(layer)
return params
def _make_train_function():
inputs = []
# shape: nb_inputs*nb_models
for model in models:
inputs += model.inputs
for model in models:
inputs += model.targets
for model in models:
inputs += model.sample_weights
if model.uses_learning_phase and type(K.learning_phase()) is not int:
inputs .append (K.learning_phase())
# training_updates
trainable_weights = collect_trainable_weights(model)
training_updates = None
# hack: override K.update to return a tuple (p,new_p) for each update
K_update = K.update
def _update(x, new_x):
return (x, new_x)
K.update = _update
for device in devices:
model = next(models)
with tf.device(device):
cur_training_updates = model.optimizer.get_updates(trainable_weights, model.constraints, model.total_loss)
if training_updates is None:
training_updates = cur_training_updates
else:
training_updates += cur_training_updates
# restore K.update
K.update = K_update
# in case multiple devices want to update the same tensor, e.g., the shared weight, use the averaged tensor
training_updates = _get_averaged_updates(trainable_weights, training_updates)
# collect outputs
outputs = []
for model in models:
outputs.append(model.total_loss)
outputs.append(model.metrics_tensors)
f = K.function(inputs,
outputs = outputs,
updates = training_updates,
**model._function_kwargs)
def _f (inputs):
# adapt the inputs passed from model for f
n = len(devices)
expanded_inputs = []
start = 0
end = len(model.inputs)
expanded_inputs += _expand_for_multiple_models (inputs[start:end], n)
start = end
end = start + len(model.targets)
expanded_inputs += _expand_for_multiple_models(inputs[start:end], n)
if model.sample_weights:
start = end
end = start + len(model.sample_weights)
expanded_inputs = _expand_for_multiple_models (inputs[start: end ], n)
return f(expanded_inputs + inputs[end:])
return _f
def trainable_weights(layers):
weights = []
for layer in layers:
weights += collect_trainable_weights(layer)
return weights