def __init__( self, name: str, keras_model: Model, epochs: int = None, batches_per_epoch: int = None): """ :param name: name of the training as string :param keras_model: keras model that should be saved to the training :param epochs: integer of how many epochs are trained :param: batches_per_epoch: integer of how many batches per epoch are trained """ self._keras_model = keras_model self._training = TrainingSchema(name, keras_model.get_config()) self._training.result = ResultSchema() self._training.result.max_epochs = epochs self._training.result.max_batches_per_epoch = batches_per_epoch self._training.result.status = -1 # status for setup self.id = None
# ============================================================== # optimizer = tf.keras.optimizers.Adam(lr=LEARNING_RATE) optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer( opt=optimizer, loss_scale=loss_scaler) print('Optimizer Configuration:') print('\nOptimizer `get_slot_names()`: %s\n' % str(optimizer.get_slot_names())) autoencoder = Model(inputs, decoded) autoencoder.compile(optimizer=optimizer, loss='mae', metrics=['mae']) # ============ Test Model Recreation from config ============ # model_config = autoencoder.get_config() autoencoder = Model.from_config(model_config) autoencoder.compile(optimizer=optimizer, loss='mae', metrics=['mae'], run_eagerly=False) # ============ Test Model Summary ============ # autoencoder.summary() from callback import ProgbarLogger progbar_callback = ProgbarLogger(count_mode='samples', stateful_metrics=['mae']) print("\nEvaluation Before training - At Initialization")
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None, input_data=None, expected_output=None, expected_output_dtype=None, fixed_batch_size=False): """Test routine for a layer with a single input tensor and single output tensor. Copy of the function in keras-team/keras because it's not in the public API. If we use the one from keras-team/keras it won't work with tf.keras. """ # generate input data if input_data is None: assert input_shape if not input_dtype: input_dtype = K.floatx() input_data_shape = list(input_shape) for i, e in enumerate(input_data_shape): if e is None: input_data_shape[i] = np.random.randint(1, 4) input_data = (10 * np.random.random(input_data_shape)) input_data = input_data.astype(input_dtype) else: if input_shape is None: input_shape = input_data.shape if input_dtype is None: input_dtype = input_data.dtype if expected_output_dtype is None: expected_output_dtype = input_dtype # instantiation layer = layer_cls(**kwargs) # test get_weights , set_weights at layer level weights = layer.get_weights() layer.set_weights(weights) expected_output_shape = layer.compute_output_shape(input_shape) # test in functional API if fixed_batch_size: x = Input(batch_shape=input_shape, dtype=input_dtype) else: x = Input(shape=input_shape[1:], dtype=input_dtype) y = layer(x) assert K.dtype(y) == expected_output_dtype # check with the functional API model = Model(x, y) actual_output = model.predict(input_data) actual_output_shape = actual_output.shape for expected_dim, actual_dim in zip(expected_output_shape, actual_output_shape): if expected_dim is not None: assert expected_dim == actual_dim if expected_output is not None: assert_allclose(actual_output, expected_output, rtol=1e-3) # test serialization, weight setting at model level model_config = model.get_config() custom_objects = {layer.__class__.__name__: layer.__class__} recovered_model = model.__class__.from_config(model_config, custom_objects) if model.weights: weights = model.get_weights() recovered_model.set_weights(weights) _output = recovered_model.predict(input_data) assert_allclose(_output, actual_output, rtol=1e-3) # test training mode (e.g. useful when the layer has a # different behavior at training and testing time). if has_arg(layer.call, 'training'): model.compile('rmsprop', 'mse') model.train_on_batch(input_data, actual_output) # test instantiation from layer config layer_config = layer.get_config() layer_config['batch_input_shape'] = input_shape layer = layer.__class__.from_config(layer_config) # for further checks in the caller function return actual_output
class RecurrentModel(Sequential): # INITIALIZATION def __init__(self, input, output, initial_states=None, final_states=None, readout_input=None, teacher_force=False, decode=False, output_length=None, return_states=False, state_initializer=None, **kwargs): inputs = [input] outputs = [output] state_spec = None if initial_states is not None: if type(initial_states) not in [list, tuple]: initial_states = [initial_states] state_spec = [ InputSpec(shape=K.int_shape(state)) for state in initial_states ] if final_states is None: raise Exception('Missing argument : final_states') else: self.states = [None] * len(initial_states) inputs += initial_states else: self.states = [] state_spec = [] if final_states is not None: if type(final_states) not in [list, tuple]: final_states = [final_states] assert len(initial_states) == len( final_states ), 'initial_states and final_states should have same number of tensors.' if initial_states is None: raise Exception('Missing argument : initial_states') outputs += final_states self.decode = decode self.output_length = output_length if decode: if output_length is None: raise Exception( 'output_length should be specified for decoder') kwargs['return_sequences'] = True self.return_states = return_states if readout_input is not None: self.readout = True state_spec += [Input(batch_shape=K.int_shape(outputs[0]))] self.states += [None] inputs += [readout_input] else: self.readout = False if teacher_force and not self.readout: raise Exception('Readout should be enabled for teacher forcing.') self.teacher_force = teacher_force self.model = Model(inputs, outputs) super(RecurrentModel, self).__init__(**kwargs) input_shape = list(K.int_shape(input)) if not decode: input_shape.insert(1, None) self.input_spec = InputSpec(shape=tuple(input_shape)) self.state_spec = state_spec self._optional_input_placeholders = {} if state_initializer: if type(state_initializer) not in [list, tuple]: state_initializer = [state_initializer] * self.num_states else: state_initializer += [None] * (self.num_states - len(state_initializer)) state_initializer = [ initializers.get(init) if init else initializers.get('zeros') for init in state_initializer ] self.state_initializer = state_initializer def build(self, input_shape): if type(input_shape) is list: input_shape = input_shape[0] if not self.decode: input_length = input_shape[1] if input_length is not None: input_shape = list(self.input_spec.shape) input_shape[1] = input_length input_shape = tuple(input_shape) self.input_spec = InputSpec(shape=input_shape) if type(self.model.input) is list: model_input_shape = self.model.input_shape[0] else: model_input_shape = self.model.input_shape if not self.decode: input_shape = input_shape[:1] + input_shape[2:] for i, j in zip(input_shape, model_input_shape): if i is not None and j is not None and i != j: raise Exception('Model expected input with shape ' + str(model_input_shape) + '. Received input with shape ' + str(input_shape)) if self.stateful: self.reset_states() self.built = True # STATES @property def num_states(self): model_input = self.model.input if type(model_input) is list: return len(model_input[1:]) else: return 0 def get_initial_state(self, inputs): if type(self.model.input) is not list: return [] try: batch_size = K.int_shape(inputs)[0] except: batch_size = None state_shapes = list(map(K.int_shape, self.model.input[1:])) states = [] if self.readout: state_shapes.pop() # default value for initial_readout is handled in call() for shape in state_shapes: if None in shape[1:]: raise Exception( 'Only the batch dimension of a state can be left unspecified. Got state with shape ' + str(shape)) if shape[0] is None: ndim = K.ndim(inputs) z = K.zeros_like(inputs) slices = [slice(None)] + [0] * (ndim - 1) z = z[slices] # (batch_size,) state_ndim = len(shape) z = K.reshape(z, (-1, ) + (1, ) * (state_ndim - 1)) z = K.tile(z, (1, ) + tuple(shape[1:])) states.append(z) else: states.append(K.zeros(shape)) state_initializer = self.state_initializer if state_initializer: # some initializers don't accept symbolic shapes for i in range(len(state_shapes)): if state_shapes[i][0] is None: if hasattr(self, 'batch_size'): state_shapes[i] = ( self.batch_size, ) + state_shapes[i][1:] if None in state_shapes[i]: state_shapes[i] = K.shape(states[i]) num_state_init = len(state_initializer) num_state = self.num_states assert num_state_init == num_state, 'RNN has ' + str( num_state) + ' states, but was provided ' + str( num_state_init) + ' state initializers.' for i in range(len(states)): init = state_initializer[i] shape = state_shapes[i] try: if not isinstance(init, initializers.Zeros): states[i] = init(shape) except: raise Exception( 'Seems the initializer ' + init.__class__.__name__ + ' does not support symbolic shapes(' + str(shape) + '). Try providing the full input shape (include batch dimension) for you RecurrentModel.' ) return states def reset_states(self, states_value=None): if len(self.states) == 0: return if not self.stateful: raise AttributeError('Layer must be stateful.') if not hasattr(self, 'states') or self.states[0] is None: state_shapes = list(map(K.int_shape, self.model.input[1:])) self.states = list(map(K.zeros, state_shapes)) if states_value is not None: if type(states_value) not in (list, tuple): states_value = [states_value] * len(self.states) assert len(states_value) == len( self.states), 'Your RNN has ' + str(len( self.states)) + ' states, but was provided ' + str( len(states_value)) + ' state values.' if 'numpy' not in type(states_value[0]): states_value = list(map(np.array, states_value)) if states_value[0].shape == tuple(): for state, val in zip(self.states, states_value): K.set_value(state, K.get_value(state) * 0. + val) else: for state, val in zip(self.states, states_value): K.set_value(state, val) else: if self.state_initializer: for state, init in zip(self.states, self.state_initializer): if isinstance(init, initializers.Zeros): K.set_value(state, 0 * K.get_value(state)) else: K.set_value(state, K.eval(init(K.get_value(state).shape))) else: for state in self.states: K.set_value(state, 0 * K.get_value(state)) # EXECUTION def __call__(self, inputs, initial_state=None, initial_readout=None, ground_truth=None, **kwargs): req_num_inputs = 1 + self.num_states inputs = _to_list(inputs) inputs = inputs[:] if len(inputs) == 1: if initial_state is not None: if type(initial_state) is list: inputs += initial_state else: inputs.append(initial_state) else: if self.readout: initial_state = self._get_optional_input_placeholder( 'initial_state', self.num_states - 1) else: initial_state = self._get_optional_input_placeholder( 'initial_state', self.num_states) inputs += _to_list(initial_state) if self.readout: if initial_readout is None: initial_readout = self._get_optional_input_placeholder( 'initial_readout') inputs.append(initial_readout) if self.teacher_force: req_num_inputs += 1 if ground_truth is None: ground_truth = self._get_optional_input_placeholder( 'ground_truth') inputs.append(ground_truth) assert len(inputs) == req_num_inputs, "Required " + str( req_num_inputs) + " inputs, received " + str(len(inputs)) + "." with K.name_scope(self.name): if not self.built: self.build(K.int_shape(inputs[0])) if self._initial_weights is not None: self.set_weights(self._initial_weights) del self._initial_weights self._initial_weights = None previous_mask = _collect_previous_mask(inputs[:1]) user_kwargs = kwargs.copy() if not _is_all_none(previous_mask): if 'mask' in inspect.getargspec(self.call).args: if 'mask' not in kwargs: kwargs['mask'] = previous_mask input_shape = _collect_input_shape(inputs) output = self.call(inputs, **kwargs) output_mask = self.compute_mask(inputs[0], previous_mask) output_shape = self.compute_output_shape(input_shape[0]) self._add_inbound_node(input_tensors=inputs, output_tensors=output, input_masks=previous_mask, output_masks=output_mask, input_shapes=input_shape, output_shapes=output_shape, arguments=user_kwargs) if hasattr(self, 'activity_regularizer' ) and self.activity_regularizer is not None: regularization_losses = [ self.activity_regularizer(x) for x in _to_list(output) ] self.add_loss(regularization_losses, _to_list(inputs)) return output def call(self, inputs, initial_state=None, initial_readout=None, ground_truth=None, mask=None, training=None): # input shape: `(samples, time (padded with zeros), input_dim)` # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if type(mask) is list: mask = mask[0] if self.model is None: raise Exception('Empty RecurrentModel.') num_req_states = self.num_states if self.readout: num_actual_states = num_req_states - 1 else: num_actual_states = num_req_states if type(inputs) is list: inputs_list = inputs[:] inputs = inputs_list.pop(0) initial_states = inputs_list[:num_actual_states] if len(initial_states) > 0: if self._is_optional_input_placeholder(initial_states[0]): initial_states = self.get_initial_state(inputs) inputs_list = inputs_list[num_actual_states:] if self.readout: initial_readout = inputs_list.pop(0) if self.teacher_force: ground_truth = inputs_list.pop() else: if initial_state is not None: if not isinstance(initial_state, (list, tuple)): initial_states = [initial_state] else: initial_states = list(initial_state) if self._is_optional_input_placeholder(initial_states[0]): initial_states = self.get_initial_state(inputs) elif self.stateful: initial_states = self.states else: initial_states = self.get_initial_state(inputs) if self.readout: if initial_readout is None or self._is_optional_input_placeholder( initial_readout): output_shape = K.int_shape(_to_list((self.model.output))[0]) output_ndim = len(output_shape) input_ndim = K.ndim(inputs) initial_readout = K.zeros_like(inputs) slices = [slice(None)] + [0] * (input_ndim - 1) initial_readout = initial_readout[slices] # (batch_size,) initial_readout = K.reshape(initial_readout, (-1, ) + (1, ) * (output_ndim - 1)) initial_readout = K.tile(initial_readout, (1, ) + tuple(output_shape[1:])) initial_states.append(initial_readout) if self.teacher_force: if ground_truth is None or self._is_optional_input_placeholder( ground_truth): raise Exception( 'ground_truth must be provided for RecurrentModel with teacher_force=True.' ) if K.backend() == 'tensorflow': with tf.control_dependencies(None): counter = K.zeros((1, )) else: counter = K.zeros((1, )) counter = K.cast(counter, 'int32') initial_states.insert(-1, counter) initial_states[-2] initial_states.insert(-1, ground_truth) num_req_states += 2 if len(initial_states) != num_req_states: raise ValueError('Layer requires ' + str(num_req_states) + ' states but was passed ' + str(len(initial_states)) + ' initial states.') input_shape = K.int_shape(inputs) if self.unroll and input_shape[1] is None: raise ValueError('Cannot unroll a RNN if the ' 'time dimension is undefined. \n' '- If using a Sequential model, ' 'specify the time dimension by passing ' 'an `input_shape` or `batch_input_shape` ' 'argument to your first layer. If your ' 'first layer is an Embedding, you can ' 'also use the `input_length` argument.\n' '- If using the functional API, specify ' 'the time dimension by passing a `shape` ' 'or `batch_shape` argument to your Input layer.') preprocessed_input = self.preprocess_input(inputs, training=None) constants = self.get_constants(inputs, training=None) if self.decode: initial_states.insert(0, inputs) preprocessed_input = K.zeros((1, self.output_length, 1)) input_length = self.output_length else: input_length = input_shape[1] if self.uses_learning_phase: with learning_phase_scope(0): last_output_test, outputs_test, states_test, updates = rnn( self.step, preprocessed_input, initial_states, go_backwards=self.go_backwards, mask=mask, constants=constants, unroll=self.unroll, input_length=input_length) with learning_phase_scope(1): last_output_train, outputs_train, states_train, updates = rnn( self.step, preprocessed_input, initial_states, go_backwards=self.go_backwards, mask=mask, constants=constants, unroll=self.unroll, input_length=input_length) last_output = K.in_train_phase(last_output_train, last_output_test, training=training) outputs = K.in_train_phase(outputs_train, outputs_test, training=training) states = [] for state_train, state_test in zip(states_train, states_test): states.append( K.in_train_phase(state_train, state_test, training=training)) else: last_output, outputs, states, updates = rnn( self.step, preprocessed_input, initial_states, go_backwards=self.go_backwards, mask=mask, constants=constants, unroll=self.unroll, input_length=input_length) states = list(states) if self.decode: states.pop(0) if self.readout: states.pop() if self.teacher_force: states.pop() states.pop() if len(updates) > 0: self.add_update(updates) if self.stateful: updates = [] for i in range(len(states)): updates.append((self.states[i], states[i])) self.add_update(updates, inputs) # Properly set learning phase if 0 < self.dropout + self.recurrent_dropout: last_output._uses_learning_phase = True outputs._uses_learning_phase = True if self.return_sequences: y = outputs else: y = last_output if self.return_states: return [y] + states else: return y def step(self, inputs, states): states = list(states) if self.teacher_force: readout = states.pop() ground_truth = states.pop() assert K.ndim(ground_truth) == 3, K.ndim(ground_truth) counter = states.pop() if K.backend() == 'tensorflow': with tf.control_dependencies(None): zero = K.cast(K.zeros((1, ))[0], 'int32') one = K.cast(K.zeros((1, ))[0], 'int32') else: zero = K.cast(K.zeros((1, ))[0], 'int32') one = K.cast(K.zeros((1, ))[0], 'int32') slices = [ slice(None), counter[0] - K.switch(counter[0], one, zero) ] + [slice(None)] * (K.ndim(ground_truth) - 2) ground_truth_slice = ground_truth[slices] readout = K.in_train_phase( K.switch(counter[0], ground_truth_slice, readout), readout) states.append(readout) if self.decode: model_input = states else: model_input = [inputs] + states shapes = [] for x in model_input: if hasattr(x, '_keras_shape'): shapes.append(x._keras_shape) del x._keras_shape # Else keras internals will get messed up. model_output = _to_list(self.model.call(model_input)) for x, s in zip(model_input, shapes): setattr(x, '_keras_shape', s) if self.decode: model_output.insert(1, model_input[0]) for tensor in model_output: tensor._uses_learning_phase = self.uses_learning_phase states = model_output[1:] output = model_output[0] if self.readout: states += [output] if self.teacher_force: states.insert(-1, counter + 1) states.insert(-1, ground_truth) return output, states # SHAPE, MASK, WEIGHTS def compute_output_shape(self, input_shape): if not self.decode: if type(input_shape) is list: input_shape[0] = self._remove_time_dim(input_shape[0]) else: input_shape = self._remove_time_dim(input_shape) input_shape = _to_list(input_shape) input_shape = [input_shape[0]] + [ K.int_shape(state) for state in self.model.input[1:] ] output_shape = self.model.compute_output_shape(input_shape) if type(output_shape) is list: output_shape = output_shape[0] if self.return_sequences: if self.decode: output_shape = output_shape[:1] + ( self.output_length, ) + output_shape[1:] else: output_shape = output_shape[:1] + ( self.input_spec.shape[1], ) + output_shape[1:] if self.return_states and len(self.states) > 0: output_shape = [output_shape] + list( map(K.int_shape, self.model.output[1:])) return output_shape def compute_mask(self, input, input_mask=None): mask = input_mask[0] if type(input_mask) is list else input_mask mask = mask if self.return_sequences else None mask = [mask ] + [None] * len(self.states) if self.return_states else mask return mask def set_weights(self, weights): self.model.set_weights(weights) def get_weights(self): return self.model.get_weights() # LAYER ATTRIBS @property def updates(self): return self.model.updates def add_update(self, updates, inputs=None): self.model.add_update(updates, inputs) @property def uses_learning_phase(self): return self.teacher_force or self.model.uses_learning_phase @property def _per_input_losses(self): if hasattr(self, 'model'): return getattr(self.model, '_per_input_losses', {}) else: return {} @_per_input_losses.setter def _per_input_losses(self, val): if hasattr(self, 'model'): self.model._per_input_losses = val @property def losses(self): if hasattr(self, 'model'): return self.model.losses else: return [] @losses.setter def losses(self, val): if hasattr(self, 'model'): self.model.losses = val def add_loss(self, losses, inputs=None): self.model.add_loss(losses, inputs) @property def constraints(self): return self.model.constraints @property def trainable_weights(self): return self.model.trainable_weights @property def non_trainable_weights(self): return self.model.non_trainable_weights def get_losses_for(self, inputs): return self.model.get_losses_for(inputs) def get_updates_for(self, inputs): return self.model.get_updates_for(inputs) def _remove_time_dim(self, shape): return shape[:1] + shape[2:] # SERIALIZATION def _serialize_state_initializer(self): si = self.state_initializer if si is None: return None elif type(si) is list: return list(map(initializers.serialize, si)) else: return initializers.serialize(si) def get_config(self): config = { 'model_config': self.model.get_config(), 'decode': self.decode, 'output_length': self.output_length, 'return_states': self.return_states, 'state_initializer': self._serialize_state_initializer() } base_config = super(RecurrentModel, self).get_config() config.update(base_config) return config @classmethod def from_config(cls, config, custom_objects={}): if type(custom_objects) is list: custom_objects = {obj.__name__: obj for obj in custom_objects} custom_objects.update(_get_cells()) config = config.copy() model_config = config.pop('model_config') if model_config is None: model = None else: model = Model.from_config(model_config, custom_objects) if type(model.input) is list: input = model.input[0] initial_states = model.input[1:] else: input = model.input initial_states = None if type(model.output) is list: output = model.output[0] final_states = model.output[1:] else: output = model.output final_states = None return cls(input, output, initial_states, final_states, **config) def get_cell(self, **kwargs): return RNNCellFromModel(self.model, **kwargs) def _get_optional_input_placeholder(self, name=None, num=1): if name: if name not in self._optional_input_placeholders: if num > 1: self._optional_input_placeholders[name] = [ self._get_optional_input_placeholder() for _ in range(num) ] else: self._optional_input_placeholders[ name] = self._get_optional_input_placeholder() return self._optional_input_placeholders[name] if num == 1: optional_input_placeholder = _to_list(_OptionalInputPlaceHolder( )._inbound_nodes[0].output_tensors)[0] assert self._is_optional_input_placeholder( optional_input_placeholder) return optional_input_placeholder else: y = [] for _ in range(num): optional_input_placeholder = _to_list( _OptionalInputPlaceHolder( )._inbound_nodes[0].output_tensors)[0] assert self._is_optional_input_placeholder( optional_input_placeholder) y.append(optional_input_placeholder) return y def _is_optional_input_placeholder(self, x): if hasattr(x, '_keras_history'): if isinstance(x._keras_history[0], _OptionalInputPlaceHolder): return True return False
def partial_trainability(model_keras, Layer_names, Layer_trainabliities): """ This function allows you to get partially trainable layers function takes model_keras: keras model you have loaded or created for example using model_zoo.get_model("VGG_small_4",32,3,2) Layer_names: list containing the names of the layers that should be changed. Of course these names have to be existing in model_keras Layer_trainability: list containing floats. For each layer that should be changed, provide a value between 0 and 1 (0-layer not trainable at all; 1-layer entirely trainable) Lets say you use Layer_names=['dense_3'] and Layer_trainability=[0.25] Lets assume 'dense_3' has 1000 nodes. Then two new parallell dense layers will be created. The first one gets 750 nodes, which are set to set to Trainable=False. The second dense layer has 250 nodes, which are trainable. The corresponding weights from the initial model are distributed to these layer accordingly. """ #Get the config of the current model model_config = model_keras.get_config() #Deep-Copy the config of the original model model_config_new = copy.deepcopy(model_config) if type(model_config_new) == dict: print("Model used functional API of Keras") elif type(model_config_new) == list: print( "Model used sequential API of Keras and will now be converted to functional API" ) #api = "Sequential" #Convert to functional API input_layer = Input(batch_shape=model_keras.layers[0].input_shape) prev_layer = input_layer for layer in model_keras.layers: prev_layer = layer(prev_layer) model_keras = Model([input_layer], [prev_layer]) #Now we have functional API :) #Get the model config for the converted model model_config = model_keras.get_config() model_config_new = copy.deepcopy(model_config) else: print("Unknown format for model config") #return Layer_nontrainability = list(1 - np.array(Layer_trainabliities)) del Layer_trainabliities #Original names of the layers Layer_names_orig = [ model_keras.layers[i].name for i in range(len(model_keras.layers)) ] #Now we are starting to loop over all the requested changes: for i in range(len(Layer_names)): nontrainability = Layer_nontrainability[i] layer_name = Layer_names[i] layer_names_list = [ model_config_new["layers"][it]['name'] for it in range(len(model_config_new["layers"])) ] layer_index = int( np.where(np.array(layer_names_list) == layer_name)[0]) layer_config = model_config_new['layers'][layer_index] layer_type = layer_config["class_name"] if layer_type == "Dense": split_property = "units" #'units' are the number of nodes in dense layers elif layer_type == "Conv2D": split_property = "filters" nr_nodes = layer_config["config"][split_property] nr_const = int(np.round(nontrainability * nr_nodes)) #get a config for the non-trainable part layer_config_const = copy.deepcopy(layer_config) layer_name_const = layer_config_const['config']["name"] + "_1" layer_config_const["config"][split_property] = nr_const layer_config_const["config"]["trainable"] = False #not trainable layer_config_const["config"]["name"] = layer_name_const #rename layer_config_const["name"] = layer_name_const #rename #get a config for the rest (trainable part) layer_config_rest = copy.deepcopy(layer_config) #this part will only exist if nr_nodes-nr_const>0: if nr_nodes - nr_const > 0: layer_name_rest = layer_config_rest["config"]["name"] + "_2" layer_config_rest["config"][split_property] = nr_nodes - nr_const layer_config_rest["config"]["name"] = layer_name_rest #rename layer_config_rest["name"] = layer_name_rest #rename #Assemble a config for a corresponding concatenate layer inbound_1 = [layer_name_const, 0, 0, {}] inbound_2 = [layer_name_rest, 0, 0, {}] inbound_nodes = [inbound_1, inbound_2] layer_name = layer_config['config'][ "name"] #Call it like the initial layer that has been there. This will allow other layers to correctly conncet like before #'concatenate_'+layer_config["config"]["name"] config_conc = {"axis": -1, "name": layer_name, "trainable": True} conc_dict = { "class_name": 'Concatenate', "config": config_conc, "inbound_nodes": [inbound_nodes], "name": layer_name } #insert these layers into the config at Index: layer_index layerlist = model_config_new["layers"] #Replace existing layer with the const part layerlist[layer_index] = layer_config_const #After that insert the rest part layerlist.insert(layer_index + 1, layer_config_rest) #After that insert the concatenate layer layerlist.insert(layer_index + 2, conc_dict) #Update the config with the new layers model_config_new["layers"] = layerlist #Build the model using this updated config model_keras_new = Model.from_config(model_config_new) #Compilation might not be required. model_keras_new.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) iterator = 0 for layer_name in Layer_names_orig: #for all layers of the original model layer = model_keras.get_layer(layer_name) if layer_name not in Layer_names: #if this layer was not subjected to change in the new model... #Straight forward: get weights of the orignal model weights = layer.get_weights() #And put them into the new model model_keras_new.get_layer(layer_name).set_weights(weights) else: #Otherwise the layer was changed layer_type = layer.__class__.__name__ layer_config = layer.get_config() weights = layer.get_weights() #Get the original weights trainability = Layer_nontrainability[iterator] iterator += 1 if layer_type == "Dense": split_property = "units" #'units' are the number of nodes in dense layers elif layer_type == "Conv2D": split_property = "filters" nr_nodes = layer_config[split_property] nr_const = int(np.round(trainability * nr_nodes)) #Constant part if layer_type == "Dense": #Put the former weights into the layer weights_const = list(np.copy(weights)) weights_const[0] = weights_const[0][:, 0:nr_const] weights_const[1] = weights_const[1][0:nr_const] elif layer_type == "Conv2D": #Put the former weights into the layer weights_const = list(np.copy(weights)) weights_const[0] = weights_const[0][:, :, :, 0:nr_const] weights_const[1] = weights_const[1][0:nr_const] else: print("Unknown layer type") #return layer_name_const = layer_name + "_1" #rename model_keras_new.get_layer(layer_name_const).set_weights( weights_const) #Rest trainable part #this part will only exist if nr_nodes-nr_const>0: if nr_nodes - nr_const > 0: if layer_type == "Dense": #Get the weights weights_rest = list(np.copy(weights)) weights_rest[0] = weights_rest[0][:, nr_const:] weights_rest[1] = weights_rest[1][nr_const:] if layer_type == "Conv2D": #Get the weights weights_rest = list(np.copy(weights)) weights_rest[0] = weights_rest[0][:, :, :, nr_const:] weights_rest[1] = weights_rest[1][nr_const:] layer_name_rest = layer_name + "_2" #rename model_keras_new.get_layer(layer_name_rest).set_weights( weights_rest) return model_keras_new
w_gp = w_gp[:, -1] w_gp = np.reshape(w_gp, [nx, ny]) w_gp = pbc(w_gp) plot_true_gp(t, aTest[:, :], aGPtest[:, :], f'modes_test_{ReTest}_{Re[pref]}.png') #%% LSTM [Fully Nonintrusive] # testing aML_all_seeds = np.zeros((num_ensembles, ns + 1, nr)) for j in range(1, num_ensembles + 1): print(j) model = load_model(f'nirom_model_{j*10}.hd5') model.get_config() testing_set = np.zeros((ns + 1, nr + 1)) testing_set[:, 0] = ReTest testing_set[:, 1:] = aTest m, n = aTest.shape xtest = np.zeros((1, lookback, nr + 1)) xtest_gp = np.zeros((1, lookback, nr)) aML = np.zeros((ns + 1, nr)) # Initializing for i in range(lookback): xtest[0, i, :] = testing_set[i, :].reshape(1, -1) xtest_gp[0, i, :] = aGPtest[i, :].reshape(1, -1)