def IndRNN(hidden_size, return_sequences=False, **kwargs): cells = [IndRNNCell(hidden_size, **kwargs), IndRNNCell(hidden_size, **kwargs)] return RNN(cells, return_sequences)
def print_model(filename, n_hidden, lstm_layer, dropout, type_name): # Get original dataset x, y = get_dataset(type_name) # Fill original class type with the label 1 y = [1 for label in y] # Get negative dataset neg_x, neg_y = get_dataset("not-" + type_name) # Fill original class type with the label 1 neg_y = [0 for label in neg_y] x.extend(neg_x) y.extend(neg_y) # Flatten X coodinates and filter x = np.array(x) _x = [] _y = [] for idx, data in enumerate(x): data = [np.reshape(np.array(frames), (28)).tolist() for frames in data] _x.append(data) _y.append(y[idx]) x = _x y = _y # Split to training and test dataset x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3) x_train = np.array(x_train) x_test = np.array(x_test) y_train = np.array(y_train) y_test = np.array(y_test) # Define training parameters n_output = 1 # Make LSTM Layer # Pair of lstm cell initialization through loop lstm_cells = [ LSTMCell(n_hidden, activation='relu', use_bias=True, unit_forget_bias=1.0) for _ in range(lstm_layer) ] stacked_lstm = StackedRNNCells(lstm_cells) # Decaying learning rate learning_rate = 1e-2 lr_schedule = PolynomialDecay(initial_learning_rate=learning_rate, decay_steps=10, end_learning_rate=0.00001) optimizer = Adam(learning_rate=lr_schedule) # Initiate model model = Sequential() model.add(InputLayer(input_shape=x_train[0].shape)) model.add(RNN(stacked_lstm)) # model.add(LSTM( # n_hidden, # activation='relu', # use_bias=True, # unit_forget_bias = 1.0, # input_shape=x_train[0].shape, # return_sequences=True)) # for _ in range(lstm_layer-2): # model.add(LSTM( # n_hidden, # activation='relu', # use_bias=True, # unit_forget_bias = 1.0, # return_sequences=True)) # model.add(LSTM( # n_hidden, # activation='relu', # use_bias=True, # unit_forget_bias = 1.0)) model.add(Dropout(dropout)) model.add( Dense(n_output, activation='sigmoid', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Train model model.fit(x_train, y_train, epochs=1, batch_size=1000, shuffle=True, validation_data=(x_test, y_test), validation_split=0.4) # Print model with open(f'{filename}.txt', 'w') as f: model.summary(print_fn=lambda x: f.write(x + '\n')) plot_model(model, to_file=f'{filename}.png', show_shapes=True, show_layer_names=True)
def __init__(self, params): self.params = params self.rnn_cell = GRUCell(self.params['rnn_size']) self.rnn = RNN(self.rnn_cell, return_state=True, return_sequences=True) self.dense = Dense(units=1) self.attention = Attention(hidden_size=32, num_heads=2, attention_dropout=0.8)
def create_pinn_model(a, b, Pu, grid_array_aSKF, bounds_aSKF, table_shape_aSKF, grid_array_kappa, bounds_kappa, table_shape_kappa, grid_array_etac, bounds_etac, table_shape_etac, d0RNN, batch_input_shape, selectdKappa, selectCycle, selectLoad, selectBTemp, myDtype, return_sequences=False, unroll=False): batch_adjusted_shape = (batch_input_shape[2] + 1, ) #Adding states placeHolder = Input(shape=(batch_input_shape[2] + 1, )) #Adding states filterdKappaLayer = inputsSelection(batch_adjusted_shape, selectdKappa)(placeHolder) filterCycleLayer = inputsSelection(batch_adjusted_shape, selectCycle)(placeHolder) filterLoadLayer = inputsSelection(batch_adjusted_shape, selectLoad)(placeHolder) filterBTempLayer = inputsSelection(batch_adjusted_shape, selectBTemp)(placeHolder) physicalSpaceInvLoadLayer = Lambda(lambda x: (1 / (10**x)))( filterLoadLayer) xvalKappaLayer = Concatenate(axis=-1)( [filterBTempLayer, filterdKappaLayer]) kappaLayer = TableInterpolation(table_shape=table_shape_kappa, dtype=myDtype, trainable=False) kappaLayer.build(input_shape=xvalKappaLayer.shape) kappaLayer.set_weights([grid_array_kappa, bounds_kappa]) kappaLayer = kappaLayer(xvalKappaLayer) xvalEtacLayer = Concatenate(axis=-1)([kappaLayer, filterdKappaLayer]) etacLayer = TableInterpolation(table_shape=table_shape_etac, dtype=myDtype, trainable=False) etacLayer.build(input_shape=xvalEtacLayer.shape) etacLayer.set_weights([grid_array_etac, bounds_etac]) etacLayer = etacLayer(xvalEtacLayer) xvalLayer1 = Lambda(lambda x: Pu * x)(etacLayer) xvalLayer2 = Multiply()([xvalLayer1, physicalSpaceInvLoadLayer]) xvalLayer = Concatenate(axis=-1)([xvalLayer2, kappaLayer]) aSKFLayer = TableInterpolation(table_shape=table_shape_aSKF, dtype=myDtype, trainable=False) aSKFLayer.build(input_shape=xvalLayer.shape) aSKFLayer.set_weights([grid_array_aSKF, bounds_aSKF]) aSKFLayer = aSKFLayer(xvalLayer) inverseaSKFLayer = Lambda(lambda x: (1 / x))(aSKFLayer) sn_input_shape = (batch_input_shape[0], batch_input_shape[2]) SNLayer = SNCurve(input_shape=sn_input_shape, dtype=myDtype, trainable=False) SNLayer.build(input_shape=sn_input_shape) SNLayer.set_weights([np.asarray([a, b], dtype=SNLayer.dtype)]) SNLayer = SNLayer(filterLoadLayer) multiplyLayer1 = Multiply()([SNLayer, filterCycleLayer]) multiplyLayer2 = Multiply()([multiplyLayer1, inverseaSKFLayer]) functionalModel = Model(inputs=[placeHolder], outputs=[multiplyLayer2]) "-------------------------------------------------------------------------" CDMCellHybrid = CumulativeDamageCell(model=functionalModel, batch_input_shape=batch_input_shape, dtype=myDtype, initial_damage=d0RNN) CDMRNNhybrid = RNN(cell=CDMCellHybrid, return_sequences=return_sequences, return_state=False, batch_input_shape=batch_input_shape, unroll=unroll) model = Sequential() model.add(CDMRNNhybrid) model.compile(loss='mse', optimizer=RMSprop(5e-4), metrics=['mae']) return model
class EnasMutator(Mutator): def __init__(self, model, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, cell_exit_extra_step=False, skip_target=0.4, temperature=None, branch_bias=0.25, entropy_reduction='sum'): super().__init__(model) self.tanh_constant = tanh_constant self.temperature = temperature self.cell_exit_extra_step = cell_exit_extra_step cells = [ LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers) ] self.lstm = RNN(cells, stateful=True) self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) self.max_layer_choice = 0 self.bias_dict = {} for mutable in self.mutables: if isinstance(mutable, LayerChoice): if self.max_layer_choice == 0: self.max_layer_choice = len(mutable) assert self.max_layer_choice == len(mutable), \ "ENAS mutator requires all layer choice have the same number of candidates." if 'reduce' in mutable.key: bias = [] for choice in mutable.choices: if 'conv' in str(type(choice)).lower(): bias.append(branch_bias) else: bias.append(-branch_bias) self.bias_dict[mutable.key] = tf.constant(bias) # exposed for trainer self.sample_log_prob = 0 self.sample_entropy = 0 self.sample_skip_penalty = 0 # internal nn layers self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) self.soft = Dense(self.max_layer_choice, use_bias=False) self.attn_anchor = Dense(lstm_size, use_bias=False) self.attn_query = Dense(lstm_size, use_bias=False) self.v_attn = Dense(1, use_bias=False) assert entropy_reduction in [ 'sum', 'mean' ], 'Entropy reduction must be one of sum and mean.' self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean self.cross_entropy_loss = SparseCategoricalCrossentropy( from_logits=True, reduction=Reduction.NONE) self._first_sample = True def sample_search(self): self._initialize() self._sample(self.mutables) self._first_sample = False return self._choices def sample_final(self): return self.sample_search() def _sample(self, tree): mutable = tree.mutable if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: self._choices[mutable.key] = self._sample_layer_choice(mutable) elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: self._choices[mutable.key] = self._sample_input_choice(mutable) for child in tree.children: self._sample(child) if self.cell_exit_extra_step and isinstance( mutable, MutableScope) and mutable.key not in self._anchors_hid: self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1) def _initialize(self): self._choices = {} self._anchors_hid = {} self._inputs = self.g_emb # seems the `input_shape` parameter of RNN does not work # workaround it by omitting `reset_states` for first run if not self._first_sample: self.lstm.reset_states() self.sample_log_prob = 0 self.sample_entropy = 0 self.sample_skip_penalty = 0 def _sample_layer_choice(self, mutable): logit = self.soft(self.lstm(self._inputs)) if self.temperature is not None: logit /= self.temperature if self.tanh_constant is not None: logit = self.tanh_constant * tf.tanh(logit) if mutable.key in self.bias_dict: logit += self.bias_dict[mutable.key] softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) branch_id = tf.reshape( tf.random.categorical(softmax_logit, num_samples=1), [1]) log_prob = self.cross_entropy_loss(branch_id, logit) self.sample_log_prob += self.entropy_reduction(log_prob) entropy = log_prob * tf.math.exp(-log_prob) self.sample_entropy += self.entropy_reduction(entropy) self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1]) mask = tf.one_hot(branch_id, self.max_layer_choice) return tf.cast(tf.reshape(mask, [-1]), tf.bool) def _sample_input_choice(self, mutable): query, anchors = [], [] for label in mutable.choose_from: if label not in self._anchors_hid: self._anchors_hid[label] = self.lstm(self._inputs) query.append(self.attn_anchor(self._anchors_hid[label])) anchors.append(self._anchors_hid[label]) query = tf.concat(query, axis=0) query = tf.tanh(query + self.attn_query(anchors[-1])) query = self.v_attn(query) if self.temperature is not None: query /= self.temperature if self.tanh_constant is not None: query = self.tanh_constant * tf.tanh(query) if mutable.n_chosen is None: logit = tf.concat([-query, query], axis=1) softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) skip = tf.reshape( tf.random.categorical(softmax_logit, num_samples=1), [-1]) skip_prob = tf.math.sigmoid(logit) kl = tf.reduce_sum(skip_prob * tf.math.log(skip_prob / self.skip_targets)) self.sample_skip_penalty += kl log_prob = self.cross_entropy_loss(skip, logit) skip = tf.cast(skip, tf.float32) inputs = tf.tensordot(skip, tf.concat(anchors, 0), 1) / (1. + tf.reduce_sum(skip)) self._inputs = tf.reshape(inputs, [1, 1, -1]) else: assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." logit = tf.reshape(query, [1, -1]) softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) index = tf.reshape( tf.random.categorical(softmax_logit, num_samples=1), [-1]) skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1]) # when the size is 1, tf does not accept tensor here, complaining the shape is wrong # but using a numpy array seems fine log_prob = self.cross_entropy_loss(logit, query.numpy()) self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1]) self.sample_log_prob += self.entropy_reduction(log_prob) entropy = log_prob * tf.exp(-log_prob) self.sample_entropy += self.entropy_reduction(entropy) assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates, mutable.n_chosen) return tf.cast(skip, tf.bool)
h = K.dot(inputs, self.kernel) output = h + K.dot(prev_output, self.recurrent_kernel) return output, [output] # In[9]: # Let's use this cell in a RNN layer: cell = MinimalRNNCell(HIDDEN_SIZE) # In[10]: K.clear_session() # keras modeling model = Sequential() model.add(RNN(cell, input_shape=(TIMESTEPS, 8), return_sequences=False)) model.add(Dense(1)) model.compile(loss='mae', optimizer='adam') # In[11]: model.summary() # In[12]: history = model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, verbose=1, validation_data=(x_val, y_val))
name='recurrent_kernel') self.bias = self.add_weight(shape=(self.units * 4, ), name='bias') self.built = True def call(self, inputs, states): prev_output = states[0] h = K.dot(inputs, self.kernel) output = h + K.dot(prev_output, self.recurrent_kernel) return output, [output] model = Sequential() model.add(Input(shape=(n_timesteps, n_features))) cell = MinimalRNNCell(128) model.add(RNN(cell)) model.add(Dense(100, activation='relu')) model.add(Dense(n_outputs, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit(trainX, trainy, validation_split=0.1, epochs=epochs, batch_size=batch_size, verbose=verbose) _, accuracy = model.evaluate(testX, testy, batch_size=batch_size, verbose=0) score = accuracy * 100.0
def create_model(n_output, is_training=True): input_1 = Input(shape=(224, 224, 3), name='input_1') input_2 = Input(shape=(224, 224, 8), name='input_2') # base_1 = MobileNetV2(weights='imagenet',include_top=False, input_shape=(224, 224, 3)) # out_1 = base_1.get_layer('block_15_add').output # (None, 7, 7, 160) # merge input, cause rknn does not support different channel number for multiple inputs # input_ = Input(shape=(224, 224, 3+8), name='input_') # input_1 = Lambda(lambda x: x[:,:,:,0:3])(input_) # input_2 = Lambda(lambda x: x[:,:,:,3:8+3])(input_) base_1 = get_mobilenet_base(input_1, input_shape=(224, 224, 3), with_weights=True) out_1 = base_1.output out_1 = GlobalAveragePooling2D()(out_1) # (None, 1024) out_1 = layers.Reshape((1, 1024), name='block15_reshape')(out_1) # base_2 = s_model(input_shape=(224, 224, 8)) # out_2 = base_2.output # (None, 8, 49) # out_2 = LSTM(n_neurons, name='s_lstm')(out_2) # CuDNNLSTM is alternative base_2 = get_mobilenet_base(input_2, input_shape=(224, 224, 8), with_weights=False) out_2 = base_2.output # out_2 = layers.Flatten(name='s6_flatten')(out_2) out_2 = layers.Permute( (3, 1, 2), name='s6_permute')(out_2) # rknn does not support this op out_2 = layers.Reshape((8, 1024), name='s6_reshape')(out_2) # out_2 = layers.Permute((2,1), name='s6_permute')(out_2) # out_2 = GlobalAveragePooling2D()(out_2) # (None, 160) # merge two stream out_ = concatenate([out_1, out_2], axis=1) # (None, 9, 1024) # out_ = LSTM(n_neurons, return_sequences=False, name='s_lstm')(out_) # CuDNNLSTM is alternative # out_ = Flatten()(out_) # quantize error cell = LSTMCell(n_neurons, name='lstm_cell') out_ = RNN(cell=cell, unroll=True, name='rnn')(out_) # quantize error # out_ = layers.SimpleRNN(n_neurons)(out_) # quantize error # out_ = MyLSTM()(out_) # out_ = Flatten()(out_) # lstm_cell takes an input argument in call # out_ = layers.Lambda(lambda t: tf.unstack(t, axis=1))(out_) # out_ = layers.Lambda(lambda t: lstm_func(t))(out_) # out_ = layers.Lambda(lambda t: tf.stack(t))(out_) # out_ = layers.Lambda(lambda t: tf.transpose(t, perm=[1, 0, 2]))(out_) # out_ = Flatten()(out_) out_ = Dense(64, activation='relu')(out_) out_ = Dropout(.5)(out_) out_ = Dense(32, activation='relu')(out_) out_ = Dropout(.5)(out_) out_ = Dense(n_output, activation='softmax', name='s_out')(out_) if is_training is not True: out_ = tf.maximum(out_, -1e27, name='s_max') model = Model(inputs=[input_1, input_2], outputs=[out_]) # model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['acc']) # model.compile(Adam(lr=.0001), loss='sparse_categorical_crossentropy', metrics=['acc']) # model.compile(RMSprop(learning_rate=0.001, rho=0.9), loss='sparse_categorical_crossentropy', metrics=['acc']) # model.summary() return model # create_model_pretrain(6)
def train(type_name, filename, n_hidden, lstm_layer, dropout, epoch, batch_size, x, y): # Make filename date_string = datetime.now().isoformat().replace(':', '.') filename = f'{filename} k-fold results {date_string}' # Create file and write CSV header write_header(filename) body = {} body['n_hidden'] = n_hidden body['lstm_layer'] = lstm_layer body['dropout'] = dropout body['epoch'] = epoch body['batch_size'] = batch_size # Initialize total accuracy variable and number of K-Fold splits total = 0 n_splits = 5 # Initialize K Fold skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1) k_fold_index = 1 x = np.array(x) y = np.array(y) t_start = time.time() for train_index, test_index in skf.split(x, y): # Initialize training sets x_train = x[train_index] y_train = y[train_index] x_test = x[test_index] y_test = y[test_index] # Define training parameters n_output = 1 # Make LSTM Layer # Pair of lstm cell initialization through loop lstm_cells = [LSTMCell( n_hidden, activation='relu', use_bias=True, unit_forget_bias = 1.0 ) for _ in range(lstm_layer)] stacked_lstm = StackedRNNCells(lstm_cells) # Decaying learning rate learning_rate = 1e-2 lr_schedule = PolynomialDecay( initial_learning_rate=learning_rate, decay_steps=10, end_learning_rate= 0.00001 ) optimizer = Adam(learning_rate = lr_schedule) # Initiate model model = Sequential() model.add(RNN(stacked_lstm)) model.add(Dropout(dropout)) model.add(Dense(n_output, activation='sigmoid', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Train model fit_start = time.time() model.fit(x_train, y_train, epochs=epoch, batch_size=batch_size, shuffle = True, validation_data = (x_test, y_test), validation_split = 0.4, callbacks=[ForceGarbageCollection()]) # Print model stats print(model.summary()) # Find accuracy _, accuracy = model.evaluate(x_test, y_test) accuracy *= 100 total += accuracy body[f'k-fold {k_fold_index}'] = "{:.2f}".format(accuracy) body[f'k-fold {k_fold_index} time'] = float(time.time() - fit_start) print('Accuracy: %.2f' % (accuracy)) k_fold_index += 1 # UNTUK SELANJUTNYA, DIBUAT TRY EXCEPT UNTUK SETIAP BLOCK BERBEDA # SEPERTI SAAT PREDICT ATAU SAAT OLAH DATA ATAUPUN SAAT CEK AKURASI # AGAR GAMPANG PINPOINT MASALAH. # Write iterations body['seconds_to_finish'] = float(time.time() - t_start) body['exercise name'] = type_name body['avg'] = "{:.2f}".format(total/n_splits) write_body(filename, body)
# Convert Types for GPU mu = mu.astype(dtype='float32') si = si.astype(dtype='float32') if verbose: print("mu init:", mu) print("si init:", si) return mu, si # Let's use this cell in a RNN layer: cell = MinimalRNNFocusedCell(32) x = keras.Input((None, 5)) layer = RNN(cell) y = layer(x) # Here's how to use the cell to build a stacked RNN: cells = [MinimalRNNFocusedCell(32), MinimalRNNFocusedCell(64)] x = keras.Input((None, 5)) layer = RNN(cells) y = layer(x) numpy.random.seed(7) # load the dataset but only keep the top n words, zero the rest top_words = 5000 (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words) # truncate and pad input sequences max_review_length = 500
def Model(X_shape=None, y_shape=None, n_layers=1, units_layer=100, dropout=0.2, architecture='LSTM'): '''Function to produce a Recurrent Neural Network customising some settings. Parameters ---------- X_shape : array This array is contains the shape of the input data. y_shape: array This array contains the shape of the output data. n_layers : int Depth of the Recurrent Neural Network. units_layer : int number of cells or hidden states produced within the layer of the Recurrent Neural Network dropout : float Fraction of the conectios to dropout during trainning architecture : string Recurrent Neural Network architecture to apply. Architectures LSTM, GRU and SimpleRNN (alias sRNN) from Keras implementation are available. From RNNNovelARchitectures module Fusion RNN (alias fRNN), Neuro biomodulated recurrent cell (alias nBRC), Biomodulated recurrent cell (alias BRC), Minimal gated unit (alias MGU) and its variants (MGU1, MGU2 and MGU3). Returns ------- model : tf.keras.utils.Sequence Recurrent Neural Network object based on the given settings. ''' model = Sequential() input_shape = (X_shape[1], X_shape[2]) novel_architecture = architecture not in ['LSTM', 'GRU', 'sRNN'] for layer in range(n_layers): first_layer = layer == 0 only_layer = n_layers == 1 mid_layer = layer != 0 and layer != n_layers - 1 if only_layer: args0 = {'input_shape': input_shape} print('Only layer, {}'.format(args0)) elif first_layer: args0 = {'return_sequences': True, 'input_shape': input_shape} print('First layer, {}'.format(args0)) elif mid_layer and not novel_architecture: args0 = { 'return_sequences': True, 'dropout': dropout, 'recurrent_dropout': dropout } print('middle, {}'.format(args0)) elif mid_layer and novel_architecture: args0 = {'return_sequences': True} print('middle, {} + in and out Dropout layers'.format(args0)) else: args0 = {'return_sequences': False} print('last one, {}'.format(args0)) print(f'{architecture}:', *args0) if novel_architecture: try: # Solution from: https://www.tutorialspoint.com/How-to-convert-a-string-to-a-Python-class-object architecture_object = getattr(sys.modules[__name__], architecture) rnn = RNN(architecture_object(units_layer), **args0) except TypeError: print('Unknown architecture.') else: architecture_object = getattr(sys.modules[__name__], architecture) rnn = architecture_object(units_layer, **args0) if mid_layer and novel_architecture: model.add(Dropout(dropout)) model.add(rnn) model.add(Dropout(dropout)) else: model.add(rnn) model.add(Dropout(dropout)) print('y_shape: ', y_shape, '\tX_shape: ', X_shape) model.add(Dense(y_shape[1], activation='softmax')) return model
import numpy as np from tensorflow.keras.models import Sequential from tensorflow.keras.activations import get as get_activation from tensorflow.keras.layers import SimpleRNNCell, RNN, Layer from tensorflow.keras.layers.experimental import LayerNormalization class SimpleRNNCellWithLayerNorm(SimpleRNNCell): def __init__(self, units, **kwargs): self.activation = get_activation(kwargs.get("activation", "tanh")) kwargs["activation"] = None super().__init__(units, **kwargs) self.layer_norm = LayerNormalization() def call(self, inputs, states): outputs, new_states = super().call(inputs, states) norm_out = self.activation(self.layer_norm(outputs)) return norm_out, [norm_out] model = Sequential([ RNN(SimpleRNNCellWithLayerNorm(20), return_sequences=True, input_shape=[None, 20]), RNN(SimpleRNNCellWithLayerNorm(5)), ]) model.compile(loss="mse", optimizer="sgd") X_train = np.random.randn(100, 50, 20) Y_train = np.random.randn(100, 5) history = model.fit(X_train, Y_train, epochs=2)
def build_wallscheid_lptn(x_shape, loss_weights=None, verbose=True, loss='mse', # LPTN params cap0=np.log10(1.0666e4), cap1=np.log10(6.5093e3), cap2=np.log10(0.437127e3), cap3=np.log10(3.5105e3), const_Rs0=np.log10(0.0375), const_Rs1=np.log10(0.0707), const_Rs2=np.log10(0.0899), lin_Rs_slope=-54e-4, lin_Rs_bias=np.log10(18e-3), exp_Rs_magn0=1.7275, exp_Rs_magn1=0.8486, exp_Rs_magn2=0.6349, exp_Rs_b0=0.1573, exp_Rs_b1=0.1428, exp_Rs_b2=0.1184, exp_Rs_a0=0.3039, exp_Rs_a1=0.2319, exp_Rs_a2=0.1205, bipoly_Rs_magn=np.log10(0.3528), bipoly_Rs_a=-0.2484, bipoly_Rs_b=0.0276, bipoly_Rs_c=0.3331, ploss_Rdc=np.log10(14.6e-3), ploss_alpha_cu=20e-4, ploss_alpha_ac_1=0.562, ploss_alpha_ac_2=0.2407, ploss_beta_cu=2.5667, ploss_k_1_0=0.5441, ploss_k_1_1=78e-4, ploss_k_1_2=0.0352, ploss_k_1_3=-0.7438, ploss_k_2=0.8655, ploss_alpha_fe=-28e-4, schlepp_factor=1.4762, ): """O. Wallscheid, "Ein Beitrag zur thermischen Ausnutzung permanenterregter Synchronmotoren in automobilen Traktionsanwendungen", Dissertation 2017 """ x = Input(batch_input_shape=x_shape, ragged=False) out = RNN(WallscheidLPTNCell(cap0, cap1, cap2, cap3, const_Rs0, const_Rs1, const_Rs2, lin_Rs_slope, lin_Rs_bias, exp_Rs_magn0, exp_Rs_magn1, exp_Rs_magn2, exp_Rs_b0, exp_Rs_b1, exp_Rs_b2, exp_Rs_a0, exp_Rs_a1, exp_Rs_a2, bipoly_Rs_magn, bipoly_Rs_a, bipoly_Rs_b, bipoly_Rs_c, ploss_Rdc, ploss_alpha_cu, ploss_alpha_ac_1, ploss_alpha_ac_2, ploss_beta_cu, ploss_k_1_0, ploss_k_1_1, ploss_k_1_2, ploss_k_1_3, ploss_k_2, ploss_alpha_fe, schlepp_factor ), return_sequences=True, stateful=True, name='rnn')(x) if loss_weights is not None: loss_weights = list(loss_weights) model = Model(x, out) model.compile(loss=loss, sample_weight_mode='temporal', loss_weights=loss_weights, run_eagerly=False) if verbose: model.summary() return model
output = prev_output + 1. return output, [output] batch_size = 3 time_step = 100 # interation feature_size = 8 x0 = Input((feature_size)) layer0 = RepeatVector(time_step) tmp = layer0(x0) x1 = Input((feature_size)) cell = RNNCell(feature_size) layer1 = RNN(cell) y = layer1(inputs=tmp, initial_state=x1) model = Model([x0, x1], y) print(model.summary()) input0 = np.zeros((batch_size, feature_size), np.float32) input1 = np.zeros((batch_size, feature_size), np.float32) input1.fill(2.) print(input0) print(input1) output0 = model.predict([input0, input1]) print(output0)
class LMU(Layer): """ A layer of trainable low-dimensional delay systems. Each unit buffers its encoded input by internally representing a low-dimensional (i.e., compressed) version of the input window. Nonlinear decodings of this representation provide computations across the window, such as its derivative, energy, median value, etc (*). Note that decoders can span across all of the units. By default the window lengths are trained via backpropagation, as well as the encoding and decoding weights. Optionally, the state-space matrices that implement the low-dimensional delay system can be trained as well, but these are shared across all of the units in the layer. Based on the occurrence of the recurrent connections, this layer will choose different implementations of evaluating the delay system. If any recurrent connections are enabled, evaluation will occur sequentially with a Keras RNN layer using the ``LMUCell`` cell class. If all recurrent connections are disabled, evaluation of the delay system will be computed as the convolution of the input sequence with the impulse response of the LMU cell, using the ``LMUCellFFT`` cell class. (*) Voelker and Eliasmith (2018). Improving spiking dynamical networks: Accurate delays, higher-order synapses, and time cells. Neural Computation, 30(3): 569-609. (*) Voelker and Eliasmith. "Methods and systems for implementing dynamic neural networks." U.S. Patent Application No. 15/243,223. Filing date: 2016-08-22. """ def __init__( self, units, order, theta, # relative to dt=1 method="zoh", realizer=Identity(), # TODO: Deprecate? factory=LegendreDelay, # TODO: Deprecate? memory_to_memory=True, hidden_to_memory=True, hidden_to_hidden=True, trainable_input_encoders=True, trainable_hidden_encoders=True, trainable_memory_encoders=True, trainable_input_kernel=True, trainable_hidden_kernel=True, trainable_memory_kernel=True, trainable_A=False, trainable_B=False, input_encoders_initializer="lecun_uniform", hidden_encoders_initializer="lecun_uniform", memory_encoders_initializer=Constant(0), # 'lecun_uniform', input_kernel_initializer="glorot_normal", hidden_kernel_initializer="glorot_normal", memory_kernel_initializer="glorot_normal", hidden_activation="tanh", return_sequences=False, **kwargs): # Note: Setting memory_to_memory, hidden_to_memory, and hidden_to_hidden to # False doesn't actually remove the connections, but only initializes the # weights to be zero and non-trainable (when using the LMUCell). # This behaviour may change pending a future API decision. self.units = units self.order = order self.theta = theta self.method = method self.realizer = realizer self.factory = factory self.memory_to_memory = memory_to_memory self.hidden_to_memory = hidden_to_memory self.hidden_to_hidden = hidden_to_hidden self.trainable_input_encoders = trainable_input_encoders self.trainable_hidden_encoders = (trainable_hidden_encoders if hidden_to_memory else False) self.trainable_memory_encoders = (trainable_memory_encoders if memory_to_memory else False) self.trainable_input_kernel = trainable_input_kernel self.trainable_hidden_kernel = (trainable_hidden_kernel if hidden_to_hidden else False) self.trainable_memory_kernel = trainable_memory_kernel self.trainable_A = trainable_A self.trainable_B = trainable_B self.input_encoders_initializer = input_encoders_initializer self.hidden_encoders_initializer = (hidden_encoders_initializer if hidden_to_memory else Constant(0)) self.memory_encoders_initializer = (memory_encoders_initializer if memory_to_memory else Constant(0)) self.input_kernel_initializer = input_kernel_initializer self.hidden_kernel_initializer = (hidden_kernel_initializer if hidden_to_hidden else Constant(0)) self.memory_kernel_initializer = memory_kernel_initializer self.hidden_activation = hidden_activation self.return_sequences = return_sequences super().__init__(**kwargs) if self.fft_check(): self.lmu_layer = LMUCellFFT( units=self.units, order=self.order, theta=self.theta, trainable_input_encoders=self.trainable_input_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_memory_kernel=self.trainable_memory_kernel, input_encoders_initializer=self.input_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, return_sequences=self.return_sequences, ) else: self.lmu_layer = RNN( LMUCell( units=self.units, order=self.order, theta=self.theta, method=self.method, realizer=self.realizer, factory=self.factory, trainable_input_encoders=self.trainable_input_encoders, trainable_hidden_encoders=self.trainable_hidden_encoders, trainable_memory_encoders=self.trainable_memory_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_hidden_kernel=self.trainable_hidden_kernel, trainable_memory_kernel=self.trainable_memory_kernel, trainable_A=self.trainable_A, trainable_B=self.trainable_B, input_encoders_initializer=self.input_encoders_initializer, hidden_encoders_initializer=self. hidden_encoders_initializer, memory_encoders_initializer=self. memory_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, hidden_kernel_initializer=self.hidden_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, ), return_sequences=self.return_sequences, ) def call(self, inputs): """ Calls the layer with inputs. """ return self.lmu_layer.call(inputs) def build(self, input_shape): """ Initializes network parameters. """ self.lmu_layer.build(input_shape) self.built = True def fft_check(self): """ Checks if recurrent connections are enabled to automatically switch to FFT. """ # Note: Only the flags are checked here. The alternative would be to check the # weight initializers and trainiable flag settings, however it is cumbersome # to check against all initializers forms that initialize the recurrent weights # to 0. # # These flags used below exist in other LMUCell implementations, and will be # brought forward in a future API decisions. return not (self.memory_to_memory or self.hidden_to_memory or self.hidden_to_hidden) def get_config(self): """ Overrides the tensorflow get_config function. """ config = super().get_config() config.update( dict( units=self.units, order=self.order, theta=self.theta, method=self.method, factory=self.factory, memory_to_memory=self.memory_to_memory, hidden_to_memory=self.hidden_to_memory, hidden_to_hidden=self.hidden_to_hidden, trainable_input_encoders=self.trainable_input_encoders, trainable_hidden_encoders=self.trainable_hidden_encoders, trainable_memory_encoders=self.trainable_memory_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_hidden_kernel=self.trainable_hidden_kernel, trainable_memory_kernel=self.trainable_memory_kernel, trainable_A=self.trainable_A, trainable_B=self.trainable_B, input_encorders_initializer=self.input_encoders_initializer, hidden_encoders_initializer=self.hidden_encoders_initializer, memory_encoders_initializer=self.memory_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, hidden_kernel_initializer=self.hidden_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, return_sequences=self.return_sequences, )) return config
def __init__(self, params): cell = GRUCell(units=params['rnn_units']) self.rnn = RNN(cell, return_state=True, return_sequences=True) self.dense = Dense(units=2) self.activate = Activate()
def __init__( self, units, order, theta, # relative to dt=1 method="zoh", realizer=Identity(), # TODO: Deprecate? factory=LegendreDelay, # TODO: Deprecate? memory_to_memory=True, hidden_to_memory=True, hidden_to_hidden=True, trainable_input_encoders=True, trainable_hidden_encoders=True, trainable_memory_encoders=True, trainable_input_kernel=True, trainable_hidden_kernel=True, trainable_memory_kernel=True, trainable_A=False, trainable_B=False, input_encoders_initializer="lecun_uniform", hidden_encoders_initializer="lecun_uniform", memory_encoders_initializer=Constant(0), # 'lecun_uniform', input_kernel_initializer="glorot_normal", hidden_kernel_initializer="glorot_normal", memory_kernel_initializer="glorot_normal", hidden_activation="tanh", return_sequences=False, **kwargs): # Note: Setting memory_to_memory, hidden_to_memory, and hidden_to_hidden to # False doesn't actually remove the connections, but only initializes the # weights to be zero and non-trainable (when using the LMUCell). # This behaviour may change pending a future API decision. self.units = units self.order = order self.theta = theta self.method = method self.realizer = realizer self.factory = factory self.memory_to_memory = memory_to_memory self.hidden_to_memory = hidden_to_memory self.hidden_to_hidden = hidden_to_hidden self.trainable_input_encoders = trainable_input_encoders self.trainable_hidden_encoders = (trainable_hidden_encoders if hidden_to_memory else False) self.trainable_memory_encoders = (trainable_memory_encoders if memory_to_memory else False) self.trainable_input_kernel = trainable_input_kernel self.trainable_hidden_kernel = (trainable_hidden_kernel if hidden_to_hidden else False) self.trainable_memory_kernel = trainable_memory_kernel self.trainable_A = trainable_A self.trainable_B = trainable_B self.input_encoders_initializer = input_encoders_initializer self.hidden_encoders_initializer = (hidden_encoders_initializer if hidden_to_memory else Constant(0)) self.memory_encoders_initializer = (memory_encoders_initializer if memory_to_memory else Constant(0)) self.input_kernel_initializer = input_kernel_initializer self.hidden_kernel_initializer = (hidden_kernel_initializer if hidden_to_hidden else Constant(0)) self.memory_kernel_initializer = memory_kernel_initializer self.hidden_activation = hidden_activation self.return_sequences = return_sequences super().__init__(**kwargs) if self.fft_check(): self.lmu_layer = LMUCellFFT( units=self.units, order=self.order, theta=self.theta, trainable_input_encoders=self.trainable_input_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_memory_kernel=self.trainable_memory_kernel, input_encoders_initializer=self.input_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, return_sequences=self.return_sequences, ) else: self.lmu_layer = RNN( LMUCell( units=self.units, order=self.order, theta=self.theta, method=self.method, realizer=self.realizer, factory=self.factory, trainable_input_encoders=self.trainable_input_encoders, trainable_hidden_encoders=self.trainable_hidden_encoders, trainable_memory_encoders=self.trainable_memory_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_hidden_kernel=self.trainable_hidden_kernel, trainable_memory_kernel=self.trainable_memory_kernel, trainable_A=self.trainable_A, trainable_B=self.trainable_B, input_encoders_initializer=self.input_encoders_initializer, hidden_encoders_initializer=self. hidden_encoders_initializer, memory_encoders_initializer=self. memory_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, hidden_kernel_initializer=self.hidden_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, ), return_sequences=self.return_sequences, )
) data_inputs.shape expected_outputs.shape sequence_length = data_inputs.shape[1] input_dim = data_inputs.shape[2] output_dim = expected_outputs.shape[2] inputs = keras.Input(shape=(None, input_dim), dtype=tf.dtypes.float32, name="encoder_inputs") encoder = RNN( cell=[LayerNormLSTMCell(128), LayerNormLSTMCell(128)], return_sequences=True, return_state=True, ) encoder_output, *encoder_hidden = encoder(inputs) context_vector, attention_weights = keras.layers.Attention()( [encoder_hidden, encoder_output]) x = tf.expand_dims(context_vector, axis=1) decoder = RNN( cell=[LayerNormLSTMCell(128), LayerNormLSTMCell(128)], return_sequences=True, return_state=False,
def train(type_name, n_hidden): # Initialize save path save_path = "/home/kevin/projects/exercise_pose_evaluation_machine/models/lstm_model/keras/" + type_name + "/" + type_name + "_lstm_model.h5" # Get original dataset x, y = get_dataset(type_name) # Fill original class type with the label 1 y = [1 for label in y] # Get negative dataset neg_x, neg_y = get_dataset("not-" + type_name) # Fill original class type with the label 1 neg_y = [0 for label in neg_y] x.extend(neg_x) y.extend(neg_y) # Flatten X coodinates and filter x = np.array(x) _x = [] _y = [] for idx, data in enumerate(x): data = [np.reshape(np.array(frames), (28)).tolist() for frames in data] _x.append(data) _y.append(y[idx]) x = _x y = _y # Split to training and test dataset x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3) x_train = np.array(x_train) x_test = np.array(x_test) y_train = np.array(y_train) y_test = np.array(y_test) # Define training parameters n_classes = 1 # Make LSTM Layer # Pair of lstm cell initialization through loop # use_bias -> Adding bias vector on each layer, by default is True so this line could be deleted # unit_forget_bias -> lstm_cells = [ LSTMCell(n_hidden, activation='relu', use_bias=True, unit_forget_bias=1.0) for _ in range(2) ] stacked_lstm = StackedRNNCells(lstm_cells) lstm_layer = RNN(stacked_lstm) learning_rate = 1e-2 lr_schedule = PolynomialDecay(initial_learning_rate=learning_rate, decay_steps=10, end_learning_rate=0.00001) optimizer = Adam(learning_rate=lr_schedule) # Initiate model # kernel_regularizers -> regularizing weights to avoid overfit training data on layer kernel # activity_regularizer -> regularizing weights to avoid overfit training data on layer output model = Sequential() model.add(lstm_layer) model.add(Dropout(0.3)) model.add( Dense(n_classes, activation='sigmoid', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # simple early stopping es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50) # Train model # shufffle = True -> shuffle training data # validation_split -> portion of training data used for validation split # validation_data -> external data used for validation model.fit(x_train, y_train, epochs=450, batch_size=150, shuffle=True, validation_data=(x_test, y_test), validation_split=0.4, callbacks=[es]) # Print model stats print(model.summary()) print(model.get_config()) # Find accuracy _, accuracy = model.evaluate(x_test, y_test) print('Accuracy: %.2f' % (accuracy * 100)) # Save model model.save(save_path) print("Saved model!") # Generate predictions print("See prediction result") random_int = random.randint(0, len(x_test)) data = x_test[random_int] prediction = "1" if model.predict(np.array([data])) > 0.5 else "0" print("predictions result:", prediction) print("expected result: ", y_test[random_int])
def __init__(self, params): self.params = params self.rnn_cell = GRUCell(units=self.params['hidden_size']) self.rnn = RNN(self.rnn_cell, return_state=True, return_sequences=True) self.dense = Dense(units=self.params['inputdim'])
def __init__(self, num_classes, hparams): super().__init__() self.decoder_embedding = Embedding(input_dim=num_classes, output_dim=hparams.embedding_dim) self.rnn_cell = DecoderCell(num_classes, hparams) self.rnn = RNN(self.rnn_cell, return_sequences=True, return_state=True)
def _init_model(self): inputs = Input(shape=(self.input_dim_1, self.input_dim_2)) # No matter for LSTM, GRU, bidirection LSTM, final layer can not use 'return_sequences' output. for i in range(self.n_layers - 1): if i == 0: res = self._rnn_block(inputs, name_index=i) else: res = self._rnn_block(res, name_index=i) # final LSTM layer if self.use_bidierc: res = Bidirectional(LSTM(self.rnn_units))(res) elif self.use_gru: res = GRU(self.rnn_units)(res) elif self.use_lstm: res = LSTM(self.rnn_units)(res) else: res = RNN(self.rnn_units)(res) # whether or not to use Dense layer if self.use_dense: res = Dense(self.dense_units)(res) if self.use_batch: res = BatchNormalization()(res) res = Activation('relu')(res) if self.use_dropout: res = Dropout(self.drop_ratio)(res) # this is method private function to check whether or not loss is not given, then use default loss def _check_loss(model, loss, metrics, optimizer): if loss is not None: model.compile(loss=loss, metrics=[metrics], optimizer=optimizer) return model if self.n_classes == 2: # this is binary class problem. out = Dense(self.n_classes, activation='sigmoid')(res) model = Model(inputs, out) if self.loss is None: model.compile(loss='binary_crossentropy', metrics=[self.metrics], optimizer=self.optimizer) else: _check_loss(model, self.loss, self.metrics, self.optimizer) elif self.n_classes >= 2: # this is multiclass problem. out = Dense(self.n_classes, activation='softmax')(res) model = Model(inputs, out) if self.loss is None: model.compile(loss='categorical_crossentropy', metrics=[self.metrics], optimizer=self.optimizer) else: _check_loss(model, self.loss, self.metrics, self.optimizer) elif self.n_classes == -1: # this is regression problem out = Dense(1)(res) model = Model(inputs, out) if self.loss is None: model.compile(loss='mse', metrics=[self.metrics], optimizer=self.optimizer) else: _check_loss(model, self.loss, self.metrics, self.optimizer) else: raise AttributeError("Parameter 'n_classes' should be -1, 2 or up 2!") if not self.silence: print('Model structure summary:') model.summary() return model
def create_model(Bias_layer, low, up, F, beta, gamma, Co, m, a0RNN, batch_input_shape, selectbias, selectidx, selectdk, selectwk, myDtype, return_sequences=False, unroll=False): batch_adjusted_shape = (batch_input_shape[2] + 1, ) #Adding state placeHolder = Input(shape=(batch_input_shape[2] + 1, )) #Adding state filterBias = inputsSelection(batch_adjusted_shape, selectbias)(placeHolder) filterSig = inputsSelection(batch_adjusted_shape, selectidx)(placeHolder) filterdK = inputsSelection(batch_adjusted_shape, selectdk)(placeHolder) filterda = inputsSelection(batch_adjusted_shape, selectwk)(placeHolder) MLP_min = low MLP_range = up - low Bias_layer = Bias_layer(filterBias) MLP = Lambda(lambda x: ((x * MLP_range) + MLP_min))(Bias_layer) Filter = Lambda(lambda x: sign(x))(filterSig) Bias_filtered_layer = Multiply()([MLP, Filter]) dk_input_shape = filterdK.get_shape() dkLayer = StressIntensityRange(input_shape=dk_input_shape, dtype=myDtype, trainable=False) dkLayer.build(input_shape=dk_input_shape) dkLayer.set_weights([np.asarray([F], dtype=dkLayer.dtype)]) dkLayer = dkLayer(filterdK) wmInput = Concatenate(axis=-1)([dkLayer, filterda]) wm_input_shape = wmInput.get_shape() wmLayer = WalkerModel(input_shape=wm_input_shape, dtype=myDtype, trainable=False) wmLayer.build(input_shape=wm_input_shape) wmLayer.set_weights( [np.asarray([beta, gamma, Co, m], dtype=wmLayer.dtype)]) wmLayer = wmLayer(wmInput) da_layer = Add()([Bias_filtered_layer, wmLayer]) functionalModel = Model(inputs=[placeHolder], outputs=[da_layer]) "-------------------------------------------------------------------------" CDMCellHybrid = CumulativeDamageCell(model=functionalModel, batch_input_shape=batch_input_shape, dtype=myDtype, initial_damage=a0RNN) CDMRNNhybrid = RNN(cell=CDMCellHybrid, return_sequences=return_sequences, return_state=False, batch_input_shape=batch_input_shape, unroll=unroll) model = Sequential() model.add(CDMRNNhybrid) model.compile(loss=mape, optimizer=RMSprop(1e-11), metrics=['mse']) return model
np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] # build model inputs = Input(shape=(doc_length, )) embed = Embedding(vocab_size, hidden_size, input_length=doc_length) embed_input = embed(inputs) #lstm=LSTM(hidden_size, return_sequences=True)(embed_input) cell = PeepholeLSTMCell(hidden_size, implementation=1) lstm = Bidirectional(RNN(cell, return_sequences=True))(embed_input) dropout = Dropout(0.5)(lstm) meanpool = Lambda(lambda x: mean(x, axis=1))(dropout) hidden = Dense(100)(meanpool) output = Dense(1, activation='sigmoid')(hidden) model = Model(inputs=inputs, outputs=output) opt = Adam() model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy']) print(model.summary()) model.fit(x_train, y_train, epochs=epoch, verbose=1) # evaluate the model loss, accuracy = model.evaluate(x_dev, y_dev, verbose=0)
X = X_train[idx].reshape(1, 1) Y = Y_train[idx].reshape(1, NUM_WORDS) # pdb.set_trace() yield X, Y sentences_as_index = convert_sentences_idx(sentences, word_to_idx) train_generator, valid_generator = create_train_valid_sentences( sentences_as_index, NUM_WORDS) pdb.set_trace() # setup the RNN cells = [MinimalRNNCell(CELL_OUTPUTS, EMBED_SIZE, NUM_WORDS)] rnn = RNN(cells, return_sequences=False, input_shape=(None, 1)) model = Sequential() model.add( Embedding(input_dim=NUM_WORDS, output_dim=EMBED_SIZE, weights=[embedding_matrix], trainable=False)) model.add(rnn) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() checkpoint_path = "checkpoints/cp-{epoch:04d}.ckpt" checkpoint_dir = os.path.dirname(checkpoint_path)
def build_ncp_model(input_shape, config, stateful=False, batch_size=1, draw_ncp_internals=False): ## Define NCP ncp_wiring = wirings.NCP( inter_neurons=config["inter_neurons"], # Number of inter neurons command_neurons=12, # Number of command neurons motor_neurons=1, # Number of motor neurons sensory_fanout=6, # How many outgoing synapses has each sensory neuron inter_fanout=config["inter_fanout"], # How many outgoing synapses has each inter neuron recurrent_command_synapses=config["recurrent_command_synapses"], # Now many recurrent synapses are in the command neuron layer motor_fanin=8, # How many incomming syanpses has each motor neuron ) ncp_cell = LTCCell(ncp_wiring) ## Build model ## # INPUT_SHAPE = (timesteps, *input_shape) if not stateful: batch_size = None IMAGE_INPUT_SHAPE = (batch_size, None, *input_shape) STEER_INPUT_SHAPE = (batch_size, None, config["command_input_size"]) # Create CNN part of the network based on Nvidia model model_cnn = Sequential() model_cnn.add(Lambda(lambda x: x / 127.5 - 1.0, batch_input_shape=IMAGE_INPUT_SHAPE, name="image_normalization")) model_cnn.add(TimeDistributed(Conv2D(24, 5, activation='elu', strides=(2, 2)), name="convolution_1")) model_cnn.add(TimeDistributed(Conv2D(36, 5, activation='elu', strides=(2, 2)), name="convolution_2")) model_cnn.add(TimeDistributed(Conv2D(48, 5, activation='elu', strides=(2, 2)), name="convolution_3")) model_cnn.add(TimeDistributed(Conv2D(64, 3, activation='elu'), name="convolution_4")) model_cnn.add(TimeDistributed(Conv2D(16, 3, activation='elu'), name="convolution_5")) model_cnn.add(TimeDistributed(Dropout(0.3), name="convolution_dropout")) model_cnn.add(TimeDistributed(Flatten(), name="convolution_flatten")) model_cnn.add(TimeDistributed(Dense(100, activation='elu'), name="convolution_inner_dense")) model_cnn.add(TimeDistributed(Dense(config["convolution_out_size"], activation='elu'), name="convolution_output_dense")) # Create input for Steering data steer_input = Input(batch_input_shape=STEER_INPUT_SHAPE, name="command_input") model_steer = steer_input if config["use_dense_size"]: model_steer = TimeDistributed(Dense(config["use_dense_size"], activation="elu"), name="command_dense")(steer_input) model_steer = TimeDistributed(Flatten(), name="command_output_flatten")(model_steer) model_steer = Model(inputs=steer_input, outputs=model_steer) # Combine CNN part with Steering input combined = concatenate([model_cnn.output, model_steer.output], name="combine_cnn_command") # Add Recurrent NCP layer to the model z = RNN(ncp_cell, return_sequences=False, stateful=stateful, name="NCP_RNN")(combined) z = Activation("tanh", name="tanh_output_activation")(z) # model.add(TimeDistributed(Dense(1))) # Finish model model = Model(inputs=[model_cnn.inputs, model_steer.inputs], outputs=z) # Visualize NCP connections - must be at the end after model is created if draw_ncp_internals: sns.set_style("white") plt.figure(figsize=(12, 8)) legend_handles = ncp_cell.draw_graph(neuron_colors={"command": "tab:cyan"}) plt.legend(handles=legend_handles, loc="upper center", bbox_to_anchor=(1.1, 1.1)) sns.despine(left=True, bottom=True) plt.tight_layout() plt.show() return model
def __init__(self, params): self.params = params cell = GRUCell(units=self.params['rnn_size']) self.rnn = RNN(cell, return_state=True, return_sequences=True) self.dense = Dense(units=1)
def nBRU(hidden_size, return_sequences=False, **kwargs): return RNN(nBRUCell(hidden_size, **kwargs), return_sequences)
def _create_model_predictor(self, lr): y = self.y.ravel() # Initial values from statsmodels implementation if self.seasonal: l0_start = y[np.arange(y.size) % self.seasonal_period == 0].mean() lead, lag = y[self.seasonal_period: 2 * self.seasonal_period], y[:self.seasonal_period] if self.trend == "multiplicative": b0_start = ((lead - lag) / self.seasonal_period).mean() elif self.trend == "additive": b0_start = np.exp((np.log(lead.mean()) - np.log(lag.mean())) / self.seasonal_period) if self.seasonal == "multiplicative": s0_start = y[:self.seasonal_period] / l0_start elif self.seasonal == "additive": s0_start = y[:self.seasonal_period] - l0_start elif self.trend: l0_start = y[0] if self.trend == "multiplicative": b0_start = y[1] / l0_start elif self.trend == "additive": b0_start = y[1] - l0_start else: l0_start = y[0] inp_y = Input(shape=(None, 1)) inp_emb_id = Input(shape=(1,)) # Dummy ID for embeddings # (Ab)using embeddings here for initial value variables init_l0 = [Embedding(1, 1, embeddings_initializer=constant(l0_start), name="l0")(inp_emb_id)[:, 0, :]] if self.trend: init_b0 = [Embedding(1, 1, embeddings_initializer=constant(b0_start), name="b0")(inp_emb_id)[:, 0, :]] else: init_b0 = [] if self.seasonal: init_seas_emb = Embedding(1, self.seasonal_period, embeddings_initializer=RandomUniform(0.8, 1.2), name="s0") init_seas = [init_seas_emb(inp_emb_id)[:, 0, :]] else: init_seas = [] rnncell = ESRNN(self.trend, self.seasonal, self.seasonal_period) rnn = RNN(rnncell, return_sequences=True, return_state=True) out_rnn = rnn(inp_y, initial_state=init_l0 + init_b0 + init_seas) if self.trend == "multiplicative": l0_b0 = init_l0[0] * init_b0[0] elif self.trend == "additive": l0_b0 = init_l0[0] + init_b0[0] else: l0_b0 = init_l0[0] if self.seasonal == "multiplicative": l0_b0_s0 = l0_b0 * init_seas[0][:, :1] elif self.seasonal == "additive": l0_b0_s0 = l0_b0 + init_seas[0][:, :1] else: l0_b0_s0 = l0_b0 out = tf.keras.layers.concatenate([ tf.math.reduce_sum(l0_b0_s0, axis=1)[:, None, None], out_rnn[0] ], axis=1) model = Model(inputs=[inp_y, inp_emb_id], outputs=out) model.compile(Adam(lr), "mse") # predictor also outputs final state for predicting out-of-sample predictor = Model(inputs=[inp_y, inp_emb_id], outputs=[out, out_rnn[1:]]) # Assign initial seasonality weights if self.seasonal: init_seas_emb.set_weights([s0_start.reshape(init_seas_emb.get_weights()[0].shape)]) return model, predictor
def create_model(MLP_C_layer, MLP_m_layer, low_C, up_C, low_m, up_m, F, a0RNN, batch_input_shape, selectaux, selectdk, myDtype, return_sequences=False, unroll=False): batch_adjusted_shape = (batch_input_shape[2] + 1, ) #Adding state placeHolder = Input(shape=(batch_input_shape[2] + 1, )) #Adding state filterLayer = inputsSelection(batch_adjusted_shape, selectaux)(placeHolder) filterdkLayer = inputsSelection(batch_adjusted_shape, selectdk)(placeHolder) MLP_C_min = low_C MLP_C_range = up_C - low_C MLP_C_layer = MLP_C_layer(filterLayer) C_layer = Lambda(lambda x: ((x * MLP_C_range) + MLP_C_min))(MLP_C_layer) MLP_m_min = low_m MLP_m_range = up_m - low_m MLP_m_layer = MLP_m_layer(filterLayer) MLP_scaled_m_layer = Lambda(lambda x: ((x * MLP_m_range) + MLP_m_min))( MLP_m_layer) dk_input_shape = filterdkLayer.get_shape() dkLayer = StressIntensityRange(input_shape=dk_input_shape, dtype=myDtype, trainable=False) dkLayer.build(input_shape=dk_input_shape) dkLayer.set_weights([np.asarray([F], dtype=dkLayer.dtype)]) dkLayer = dkLayer(filterdkLayer) ldK_layer = Lambda(lambda x: tf.math.log(x) / (tf.math.log(tf.constant(10.))))(dkLayer) dKm_layer = Multiply()([MLP_scaled_m_layer, ldK_layer]) aux_layer = Add()([C_layer, dKm_layer]) da_layer = Lambda(lambda x: 10**(x))(aux_layer) functionalModel = Model(inputs=[placeHolder], outputs=[da_layer]) "-------------------------------------------------------------------------" CDMCellHybrid = CumulativeDamageCell(model=functionalModel, batch_input_shape=batch_input_shape, dtype=myDtype, initial_damage=a0RNN) CDMRNNhybrid = RNN(cell=CDMCellHybrid, return_sequences=return_sequences, return_state=False, batch_input_shape=batch_input_shape, unroll=unroll) model = Sequential() model.add(CDMRNNhybrid) model.compile(loss='mse', optimizer=RMSprop(learning_rate=1e-6), metrics=['mae']) return model