def _make_mv_model(self, filepath): """ Create a model with moving averaged weights. Other variables are the same as original mode. We first save original model to save its state. Then copy moving averaged weights over.""" self.model.save(filepath, overwrite=True) model2 = load_model(filepath, custom_objects=self.custom_objects) for w2, w in zip(collect_trainable_weights(model2), collect_trainable_weights(self.model)): K.set_value(w2, self.mv_trainable_weights_vals[w.name]) return model2
def test_sequential_trainable(): x = Input(shape=(20,)) dense1 = Dense(20) dense2 = Dense(10) dense3 = Dense(1) seq = sequential([ dense1, dense2, dense3, ], trainable=False) seq(x) assert collect_trainable_weights(dense1) == [] assert collect_trainable_weights(dense2) == [] assert collect_trainable_weights(dense3) == []
def test_sequential_trainable(): x = Input(shape=(20, )) dense1 = Dense(20) dense2 = Dense(10) dense3 = Dense(1) seq = sequential([ dense1, dense2, dense3, ], trainable=False) seq(x) assert collect_trainable_weights(dense1) == [] assert collect_trainable_weights(dense2) == [] assert collect_trainable_weights(dense3) == []
def _build(self): fake, _, _, g_additional_losses = self.g.run_internal_graph(self.g.inputs) real = self.d.inputs[0] data = concat([fake, real], axis=0) realness, _, _, d_additional_losses = self.d.run_internal_graph( [data] + self.d.inputs[1:]) nb_fakes = fake.shape[0] fake_realness = realness[:nb_fakes] real_realness = realness[nb_fakes:] split = 2*nb_fakes // 3 g_fake_realness = fake_realness[:split] d_fake_realness = fake_realness[split:] outputs = OrderedDict() g_loss = K.mean(K.binary_crossentropy(g_fake_realness, K.ones_like(real_realness))) outputs['g_loss'] = g_loss g_reg_loss = sum([v for v in g_additional_losses.values()]) if g_reg_loss != 0: outputs['g_reg_loss'] = g_reg_loss g_total_loss = g_loss + g_reg_loss d_loss = K.mean(K.binary_crossentropy(real_realness, K.ones_like(real_realness))) d_loss += K.mean(K.binary_crossentropy(d_fake_realness, K.zeros_like(real_realness))) outputs['d_loss'] = d_loss d_reg_loss = sum([v for v in d_additional_losses.values()]) if d_reg_loss != 0: outputs['d_reg_loss'] = d_reg_loss d_total_loss = d_loss + d_reg_loss inputs = {i.name: i for i in self.g.inputs + self.d.inputs} inputs_list = [] for name in self.input_names: inputs_list.append(inputs[name]) g_updates = self.g_optimizer.get_updates( collect_trainable_weights(self.g), self.g.constraints, g_total_loss) d_updates = self.d_optimizer.get_updates( collect_trainable_weights(self.d), self.d.constraints, d_total_loss) if self.uses_learning_phase: lr_phase = [K.learning_phase()] else: lr_phase = [] self.metrics_names = list(outputs.keys()) self._train_function = K.function(inputs_list + lr_phase, list(outputs.values()), updates=g_updates + d_updates)
def print_parameters(self): from keras.engine.training import collect_trainable_weights logger.info("total number of parameters: %s" % self.graph.count_params()) trainable_weights = collect_trainable_weights(self.graph) total = sum([K.count_params(p) for p in trainable_weights]) logger.info("number of trainable parameters: %s" % total)
def on_train_begin(self, logs={}): self.sym_trainable_weights = collect_trainable_weights(self.model) # Initialize moving averaged weights using original model values self.mv_trainable_weights_vals = { x.name: K.get_value(x) for x in self.sym_trainable_weights } if self.verbose: print('Created a copy of model weights to initialize moving' ' averaged weights.')
def print_parameters(self): from keras.engine.training import collect_trainable_weights logger.info("total number of parameters: %s" % self.graph.count_params()) trainable_weights = collect_trainable_weights(self.graph) total = sum([K.count_params(p) for p in trainable_weights]) logger.info("number of trainable parameters: %s" % total)
def XXX_get_trainable_params(self, model): params = [] for layer in model.layers: params += training.collect_trainable_weights(layer) return params
def XXX_get_trainable_params(self, model): params = [] for layer in model.layers: params += training.collect_trainable_weights(layer) return params
def _make_train_function(): inputs = [] # shape: nb_inputs*nb_models for model in models: inputs += model.inputs for model in models: inputs += model.targets for model in models: inputs += model.sample_weights if model.uses_learning_phase and type(K.learning_phase()) is not int: inputs .append (K.learning_phase()) # training_updates trainable_weights = collect_trainable_weights(model) training_updates = None # hack: override K.update to return a tuple (p,new_p) for each update K_update = K.update def _update(x, new_x): return (x, new_x) K.update = _update for device in devices: model = next(models) with tf.device(device): cur_training_updates = model.optimizer.get_updates(trainable_weights, model.constraints, model.total_loss) if training_updates is None: training_updates = cur_training_updates else: training_updates += cur_training_updates # restore K.update K.update = K_update # in case multiple devices want to update the same tensor, e.g., the shared weight, use the averaged tensor training_updates = _get_averaged_updates(trainable_weights, training_updates) # collect outputs outputs = [] for model in models: outputs.append(model.total_loss) outputs.append(model.metrics_tensors) f = K.function(inputs, outputs = outputs, updates = training_updates, **model._function_kwargs) def _f (inputs): # adapt the inputs passed from model for f n = len(devices) expanded_inputs = [] start = 0 end = len(model.inputs) expanded_inputs += _expand_for_multiple_models (inputs[start:end], n) start = end end = start + len(model.targets) expanded_inputs += _expand_for_multiple_models(inputs[start:end], n) if model.sample_weights: start = end end = start + len(model.sample_weights) expanded_inputs = _expand_for_multiple_models (inputs[start: end ], n) return f(expanded_inputs + inputs[end:]) return _f
def trainable_weights(layers): weights = [] for layer in layers: weights += collect_trainable_weights(layer) return weights