def IndRNN(hidden_size,
return_sequences=False,
**kwargs):
cells = [IndRNNCell(hidden_size, **kwargs), IndRNNCell(hidden_size, **kwargs)]
return RNN(cells, return_sequences)
def print_model(filename, n_hidden, lstm_layer, dropout, type_name):
# Get original dataset
x, y = get_dataset(type_name)
# Fill original class type with the label 1
y = [1 for label in y]
# Get negative dataset
neg_x, neg_y = get_dataset("not-" + type_name)
# Fill original class type with the label 1
neg_y = [0 for label in neg_y]
x.extend(neg_x)
y.extend(neg_y)
# Flatten X coodinates and filter
x = np.array(x)
_x = []
_y = []
for idx, data in enumerate(x):
data = [np.reshape(np.array(frames), (28)).tolist() for frames in data]
_x.append(data)
_y.append(y[idx])
x = _x
y = _y
# Split to training and test dataset
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
x_train = np.array(x_train)
x_test = np.array(x_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
# Define training parameters
n_output = 1
# Make LSTM Layer
# Pair of lstm cell initialization through loop
lstm_cells = [
LSTMCell(n_hidden,
activation='relu',
use_bias=True,
unit_forget_bias=1.0) for _ in range(lstm_layer)
]
stacked_lstm = StackedRNNCells(lstm_cells)
# Decaying learning rate
learning_rate = 1e-2
lr_schedule = PolynomialDecay(initial_learning_rate=learning_rate,
decay_steps=10,
end_learning_rate=0.00001)
optimizer = Adam(learning_rate=lr_schedule)
# Initiate model
model = Sequential()
model.add(InputLayer(input_shape=x_train[0].shape))
model.add(RNN(stacked_lstm))
# model.add(LSTM(
# n_hidden,
# activation='relu',
# use_bias=True,
# unit_forget_bias = 1.0,
# input_shape=x_train[0].shape,
# return_sequences=True))
# for _ in range(lstm_layer-2):
# model.add(LSTM(
# n_hidden,
# activation='relu',
# use_bias=True,
# unit_forget_bias = 1.0,
# return_sequences=True))
# model.add(LSTM(
# n_hidden,
# activation='relu',
# use_bias=True,
# unit_forget_bias = 1.0))
model.add(Dropout(dropout))
model.add(
Dense(n_output,
activation='sigmoid',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01)))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# Train model
model.fit(x_train,
y_train,
epochs=1,
batch_size=1000,
shuffle=True,
validation_data=(x_test, y_test),
validation_split=0.4)
# Print model
with open(f'{filename}.txt', 'w') as f:
model.summary(print_fn=lambda x: f.write(x + '\n'))
plot_model(model,
to_file=f'{filename}.png',
show_shapes=True,
show_layer_names=True)
def __init__(self, params):
self.params = params
self.rnn_cell = GRUCell(self.params['rnn_size'])
self.rnn = RNN(self.rnn_cell, return_state=True, return_sequences=True)
self.dense = Dense(units=1)
self.attention = Attention(hidden_size=32, num_heads=2, attention_dropout=0.8)
def create_pinn_model(a,
b,
Pu,
grid_array_aSKF,
bounds_aSKF,
table_shape_aSKF,
grid_array_kappa,
bounds_kappa,
table_shape_kappa,
grid_array_etac,
bounds_etac,
table_shape_etac,
d0RNN,
batch_input_shape,
selectdKappa,
selectCycle,
selectLoad,
selectBTemp,
myDtype,
return_sequences=False,
unroll=False):
batch_adjusted_shape = (batch_input_shape[2] + 1, ) #Adding states
placeHolder = Input(shape=(batch_input_shape[2] + 1, )) #Adding states
filterdKappaLayer = inputsSelection(batch_adjusted_shape,
selectdKappa)(placeHolder)
filterCycleLayer = inputsSelection(batch_adjusted_shape,
selectCycle)(placeHolder)
filterLoadLayer = inputsSelection(batch_adjusted_shape,
selectLoad)(placeHolder)
filterBTempLayer = inputsSelection(batch_adjusted_shape,
selectBTemp)(placeHolder)
physicalSpaceInvLoadLayer = Lambda(lambda x: (1 / (10**x)))(
filterLoadLayer)
xvalKappaLayer = Concatenate(axis=-1)(
[filterBTempLayer, filterdKappaLayer])
kappaLayer = TableInterpolation(table_shape=table_shape_kappa,
dtype=myDtype,
trainable=False)
kappaLayer.build(input_shape=xvalKappaLayer.shape)
kappaLayer.set_weights([grid_array_kappa, bounds_kappa])
kappaLayer = kappaLayer(xvalKappaLayer)
xvalEtacLayer = Concatenate(axis=-1)([kappaLayer, filterdKappaLayer])
etacLayer = TableInterpolation(table_shape=table_shape_etac,
dtype=myDtype,
trainable=False)
etacLayer.build(input_shape=xvalEtacLayer.shape)
etacLayer.set_weights([grid_array_etac, bounds_etac])
etacLayer = etacLayer(xvalEtacLayer)
xvalLayer1 = Lambda(lambda x: Pu * x)(etacLayer)
xvalLayer2 = Multiply()([xvalLayer1, physicalSpaceInvLoadLayer])
xvalLayer = Concatenate(axis=-1)([xvalLayer2, kappaLayer])
aSKFLayer = TableInterpolation(table_shape=table_shape_aSKF,
dtype=myDtype,
trainable=False)
aSKFLayer.build(input_shape=xvalLayer.shape)
aSKFLayer.set_weights([grid_array_aSKF, bounds_aSKF])
aSKFLayer = aSKFLayer(xvalLayer)
inverseaSKFLayer = Lambda(lambda x: (1 / x))(aSKFLayer)
sn_input_shape = (batch_input_shape[0], batch_input_shape[2])
SNLayer = SNCurve(input_shape=sn_input_shape,
dtype=myDtype,
trainable=False)
SNLayer.build(input_shape=sn_input_shape)
SNLayer.set_weights([np.asarray([a, b], dtype=SNLayer.dtype)])
SNLayer = SNLayer(filterLoadLayer)
multiplyLayer1 = Multiply()([SNLayer, filterCycleLayer])
multiplyLayer2 = Multiply()([multiplyLayer1, inverseaSKFLayer])
functionalModel = Model(inputs=[placeHolder], outputs=[multiplyLayer2])
"-------------------------------------------------------------------------"
CDMCellHybrid = CumulativeDamageCell(model=functionalModel,
batch_input_shape=batch_input_shape,
dtype=myDtype,
initial_damage=d0RNN)
CDMRNNhybrid = RNN(cell=CDMCellHybrid,
return_sequences=return_sequences,
return_state=False,
batch_input_shape=batch_input_shape,
unroll=unroll)
model = Sequential()
model.add(CDMRNNhybrid)
model.compile(loss='mse', optimizer=RMSprop(5e-4), metrics=['mae'])
return model
class EnasMutator(Mutator):
def __init__(self,
model,
lstm_size=64,
lstm_num_layers=1,
tanh_constant=1.5,
cell_exit_extra_step=False,
skip_target=0.4,
temperature=None,
branch_bias=0.25,
entropy_reduction='sum'):
super().__init__(model)
self.tanh_constant = tanh_constant
self.temperature = temperature
self.cell_exit_extra_step = cell_exit_extra_step
cells = [
LSTMCell(units=lstm_size, use_bias=False)
for _ in range(lstm_num_layers)
]
self.lstm = RNN(cells, stateful=True)
self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1
self.skip_targets = tf.constant([1.0 - skip_target, skip_target])
self.max_layer_choice = 0
self.bias_dict = {}
for mutable in self.mutables:
if isinstance(mutable, LayerChoice):
if self.max_layer_choice == 0:
self.max_layer_choice = len(mutable)
assert self.max_layer_choice == len(mutable), \
"ENAS mutator requires all layer choice have the same number of candidates."
if 'reduce' in mutable.key:
bias = []
for choice in mutable.choices:
if 'conv' in str(type(choice)).lower():
bias.append(branch_bias)
else:
bias.append(-branch_bias)
self.bias_dict[mutable.key] = tf.constant(bias)
# exposed for trainer
self.sample_log_prob = 0
self.sample_entropy = 0
self.sample_skip_penalty = 0
# internal nn layers
self.embedding = Embedding(self.max_layer_choice + 1, lstm_size)
self.soft = Dense(self.max_layer_choice, use_bias=False)
self.attn_anchor = Dense(lstm_size, use_bias=False)
self.attn_query = Dense(lstm_size, use_bias=False)
self.v_attn = Dense(1, use_bias=False)
assert entropy_reduction in [
'sum', 'mean'
], 'Entropy reduction must be one of sum and mean.'
self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean
self.cross_entropy_loss = SparseCategoricalCrossentropy(
from_logits=True, reduction=Reduction.NONE)
self._first_sample = True
def sample_search(self):
self._initialize()
self._sample(self.mutables)
self._first_sample = False
return self._choices
def sample_final(self):
return self.sample_search()
def _sample(self, tree):
mutable = tree.mutable
if isinstance(mutable,
LayerChoice) and mutable.key not in self._choices:
self._choices[mutable.key] = self._sample_layer_choice(mutable)
elif isinstance(mutable,
InputChoice) and mutable.key not in self._choices:
self._choices[mutable.key] = self._sample_input_choice(mutable)
for child in tree.children:
self._sample(child)
if self.cell_exit_extra_step and isinstance(
mutable,
MutableScope) and mutable.key not in self._anchors_hid:
self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1)
def _initialize(self):
self._choices = {}
self._anchors_hid = {}
self._inputs = self.g_emb
# seems the `input_shape` parameter of RNN does not work
# workaround it by omitting `reset_states` for first run
if not self._first_sample:
self.lstm.reset_states()
self.sample_log_prob = 0
self.sample_entropy = 0
self.sample_skip_penalty = 0
def _sample_layer_choice(self, mutable):
logit = self.soft(self.lstm(self._inputs))
if self.temperature is not None:
logit /= self.temperature
if self.tanh_constant is not None:
logit = self.tanh_constant * tf.tanh(logit)
if mutable.key in self.bias_dict:
logit += self.bias_dict[mutable.key]
softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1))
branch_id = tf.reshape(
tf.random.categorical(softmax_logit, num_samples=1), [1])
log_prob = self.cross_entropy_loss(branch_id, logit)
self.sample_log_prob += self.entropy_reduction(log_prob)
entropy = log_prob * tf.math.exp(-log_prob)
self.sample_entropy += self.entropy_reduction(entropy)
self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1])
mask = tf.one_hot(branch_id, self.max_layer_choice)
return tf.cast(tf.reshape(mask, [-1]), tf.bool)
def _sample_input_choice(self, mutable):
query, anchors = [], []
for label in mutable.choose_from:
if label not in self._anchors_hid:
self._anchors_hid[label] = self.lstm(self._inputs)
query.append(self.attn_anchor(self._anchors_hid[label]))
anchors.append(self._anchors_hid[label])
query = tf.concat(query, axis=0)
query = tf.tanh(query + self.attn_query(anchors[-1]))
query = self.v_attn(query)
if self.temperature is not None:
query /= self.temperature
if self.tanh_constant is not None:
query = self.tanh_constant * tf.tanh(query)
if mutable.n_chosen is None:
logit = tf.concat([-query, query], axis=1)
softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1))
skip = tf.reshape(
tf.random.categorical(softmax_logit, num_samples=1), [-1])
skip_prob = tf.math.sigmoid(logit)
kl = tf.reduce_sum(skip_prob *
tf.math.log(skip_prob / self.skip_targets))
self.sample_skip_penalty += kl
log_prob = self.cross_entropy_loss(skip, logit)
skip = tf.cast(skip, tf.float32)
inputs = tf.tensordot(skip, tf.concat(anchors, 0),
1) / (1. + tf.reduce_sum(skip))
self._inputs = tf.reshape(inputs, [1, 1, -1])
else:
assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS."
logit = tf.reshape(query, [1, -1])
softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1))
index = tf.reshape(
tf.random.categorical(softmax_logit, num_samples=1), [-1])
skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1])
# when the size is 1, tf does not accept tensor here, complaining the shape is wrong
# but using a numpy array seems fine
log_prob = self.cross_entropy_loss(logit, query.numpy())
self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1])
self.sample_log_prob += self.entropy_reduction(log_prob)
entropy = log_prob * tf.exp(-log_prob)
self.sample_entropy += self.entropy_reduction(entropy)
assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates,
mutable.n_chosen)
return tf.cast(skip, tf.bool)
h = K.dot(inputs, self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# In[9]:
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(HIDDEN_SIZE)
# In[10]:
K.clear_session()
# keras modeling
model = Sequential()
model.add(RNN(cell, input_shape=(TIMESTEPS, 8), return_sequences=False))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
# In[11]:
model.summary()
# In[12]:
history = model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
verbose=1,
validation_data=(x_val, y_val))
name='recurrent_kernel')
self.bias = self.add_weight(shape=(self.units * 4, ), name='bias')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs, self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
model = Sequential()
model.add(Input(shape=(n_timesteps, n_features)))
cell = MinimalRNNCell(128)
model.add(RNN(cell))
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(trainX,
trainy,
validation_split=0.1,
epochs=epochs,
batch_size=batch_size,
verbose=verbose)
_, accuracy = model.evaluate(testX, testy, batch_size=batch_size, verbose=0)
score = accuracy * 100.0
def create_model(n_output, is_training=True):
input_1 = Input(shape=(224, 224, 3), name='input_1')
input_2 = Input(shape=(224, 224, 8), name='input_2')
# base_1 = MobileNetV2(weights='imagenet',include_top=False, input_shape=(224, 224, 3))
# out_1 = base_1.get_layer('block_15_add').output # (None, 7, 7, 160)
# merge input, cause rknn does not support different channel number for multiple inputs
# input_ = Input(shape=(224, 224, 3+8), name='input_')
# input_1 = Lambda(lambda x: x[:,:,:,0:3])(input_)
# input_2 = Lambda(lambda x: x[:,:,:,3:8+3])(input_)
base_1 = get_mobilenet_base(input_1,
input_shape=(224, 224, 3),
with_weights=True)
out_1 = base_1.output
out_1 = GlobalAveragePooling2D()(out_1) # (None, 1024)
out_1 = layers.Reshape((1, 1024), name='block15_reshape')(out_1)
# base_2 = s_model(input_shape=(224, 224, 8))
# out_2 = base_2.output # (None, 8, 49)
# out_2 = LSTM(n_neurons, name='s_lstm')(out_2) # CuDNNLSTM is alternative
base_2 = get_mobilenet_base(input_2,
input_shape=(224, 224, 8),
with_weights=False)
out_2 = base_2.output
# out_2 = layers.Flatten(name='s6_flatten')(out_2)
out_2 = layers.Permute(
(3, 1, 2), name='s6_permute')(out_2) # rknn does not support this op
out_2 = layers.Reshape((8, 1024), name='s6_reshape')(out_2)
# out_2 = layers.Permute((2,1), name='s6_permute')(out_2)
# out_2 = GlobalAveragePooling2D()(out_2) # (None, 160)
# merge two stream
out_ = concatenate([out_1, out_2], axis=1) # (None, 9, 1024)
# out_ = LSTM(n_neurons, return_sequences=False, name='s_lstm')(out_) # CuDNNLSTM is alternative
# out_ = Flatten()(out_)
# quantize error
cell = LSTMCell(n_neurons, name='lstm_cell')
out_ = RNN(cell=cell, unroll=True, name='rnn')(out_)
# quantize error
# out_ = layers.SimpleRNN(n_neurons)(out_)
# quantize error
# out_ = MyLSTM()(out_)
# out_ = Flatten()(out_)
# lstm_cell takes an input argument in call
# out_ = layers.Lambda(lambda t: tf.unstack(t, axis=1))(out_)
# out_ = layers.Lambda(lambda t: lstm_func(t))(out_)
# out_ = layers.Lambda(lambda t: tf.stack(t))(out_)
# out_ = layers.Lambda(lambda t: tf.transpose(t, perm=[1, 0, 2]))(out_)
# out_ = Flatten()(out_)
out_ = Dense(64, activation='relu')(out_)
out_ = Dropout(.5)(out_)
out_ = Dense(32, activation='relu')(out_)
out_ = Dropout(.5)(out_)
out_ = Dense(n_output, activation='softmax', name='s_out')(out_)
if is_training is not True:
out_ = tf.maximum(out_, -1e27, name='s_max')
model = Model(inputs=[input_1, input_2], outputs=[out_])
# model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['acc'])
# model.compile(Adam(lr=.0001), loss='sparse_categorical_crossentropy', metrics=['acc'])
# model.compile(RMSprop(learning_rate=0.001, rho=0.9), loss='sparse_categorical_crossentropy', metrics=['acc'])
# model.summary()
return model
# create_model_pretrain(6)
def train(type_name, filename, n_hidden, lstm_layer, dropout, epoch, batch_size, x, y):
# Make filename
date_string = datetime.now().isoformat().replace(':', '.')
filename = f'{filename} k-fold results {date_string}'
# Create file and write CSV header
write_header(filename)
body = {}
body['n_hidden'] = n_hidden
body['lstm_layer'] = lstm_layer
body['dropout'] = dropout
body['epoch'] = epoch
body['batch_size'] = batch_size
# Initialize total accuracy variable and number of K-Fold splits
total = 0
n_splits = 5
# Initialize K Fold
skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1)
k_fold_index = 1
x = np.array(x)
y = np.array(y)
t_start = time.time()
for train_index, test_index in skf.split(x, y):
# Initialize training sets
x_train = x[train_index]
y_train = y[train_index]
x_test = x[test_index]
y_test = y[test_index]
# Define training parameters
n_output = 1
# Make LSTM Layer
# Pair of lstm cell initialization through loop
lstm_cells = [LSTMCell(
n_hidden,
activation='relu',
use_bias=True,
unit_forget_bias = 1.0
) for _ in range(lstm_layer)]
stacked_lstm = StackedRNNCells(lstm_cells)
# Decaying learning rate
learning_rate = 1e-2
lr_schedule = PolynomialDecay(
initial_learning_rate=learning_rate,
decay_steps=10,
end_learning_rate= 0.00001
)
optimizer = Adam(learning_rate = lr_schedule)
# Initiate model
model = Sequential()
model.add(RNN(stacked_lstm))
model.add(Dropout(dropout))
model.add(Dense(n_output,
activation='sigmoid',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01)))
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
# Train model
fit_start = time.time()
model.fit(x_train, y_train, epochs=epoch, batch_size=batch_size, shuffle = True, validation_data = (x_test, y_test), validation_split = 0.4, callbacks=[ForceGarbageCollection()])
# Print model stats
print(model.summary())
# Find accuracy
_, accuracy = model.evaluate(x_test, y_test)
accuracy *= 100
total += accuracy
body[f'k-fold {k_fold_index}'] = "{:.2f}".format(accuracy)
body[f'k-fold {k_fold_index} time'] = float(time.time() - fit_start)
print('Accuracy: %.2f' % (accuracy))
k_fold_index += 1
# UNTUK SELANJUTNYA, DIBUAT TRY EXCEPT UNTUK SETIAP BLOCK BERBEDA
# SEPERTI SAAT PREDICT ATAU SAAT OLAH DATA ATAUPUN SAAT CEK AKURASI
# AGAR GAMPANG PINPOINT MASALAH.
# Write iterations
body['seconds_to_finish'] = float(time.time() - t_start)
body['exercise name'] = type_name
body['avg'] = "{:.2f}".format(total/n_splits)
write_body(filename, body)
# Convert Types for GPU
mu = mu.astype(dtype='float32')
si = si.astype(dtype='float32')
if verbose:
print("mu init:", mu)
print("si init:", si)
return mu, si
# Let's use this cell in a RNN layer:
cell = MinimalRNNFocusedCell(32)
x = keras.Input((None, 5))
layer = RNN(cell)
y = layer(x)
# Here's how to use the cell to build a stacked RNN:
cells = [MinimalRNNFocusedCell(32), MinimalRNNFocusedCell(64)]
x = keras.Input((None, 5))
layer = RNN(cells)
y = layer(x)
numpy.random.seed(7)
# load the dataset but only keep the top n words, zero the rest
top_words = 5000
(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words)
# truncate and pad input sequences
max_review_length = 500
def Model(X_shape=None,
y_shape=None,
n_layers=1,
units_layer=100,
dropout=0.2,
architecture='LSTM'):
'''Function to produce a Recurrent Neural Network customising some settings.
Parameters
----------
X_shape : array
This array is contains the shape of the input data.
y_shape: array
This array contains the shape of the output data.
n_layers : int
Depth of the Recurrent Neural Network.
units_layer : int
number of cells or hidden states produced within the layer of the
Recurrent Neural Network
dropout : float
Fraction of the conectios to dropout during trainning
architecture : string
Recurrent Neural Network architecture to apply. Architectures LSTM,
GRU and SimpleRNN (alias sRNN) from Keras implementation are
available. From RNNNovelARchitectures module Fusion RNN (alias fRNN),
Neuro biomodulated recurrent cell (alias nBRC),
Biomodulated recurrent cell (alias BRC), Minimal gated unit (alias MGU)
and its variants (MGU1, MGU2 and MGU3).
Returns
-------
model : tf.keras.utils.Sequence
Recurrent Neural Network object based on the given settings.
'''
model = Sequential()
input_shape = (X_shape[1], X_shape[2])
novel_architecture = architecture not in ['LSTM', 'GRU', 'sRNN']
for layer in range(n_layers):
first_layer = layer == 0
only_layer = n_layers == 1
mid_layer = layer != 0 and layer != n_layers - 1
if only_layer:
args0 = {'input_shape': input_shape}
print('Only layer, {}'.format(args0))
elif first_layer:
args0 = {'return_sequences': True, 'input_shape': input_shape}
print('First layer, {}'.format(args0))
elif mid_layer and not novel_architecture:
args0 = {
'return_sequences': True,
'dropout': dropout,
'recurrent_dropout': dropout
}
print('middle, {}'.format(args0))
elif mid_layer and novel_architecture:
args0 = {'return_sequences': True}
print('middle, {} + in and out Dropout layers'.format(args0))
else:
args0 = {'return_sequences': False}
print('last one, {}'.format(args0))
print(f'{architecture}:', *args0)
if novel_architecture:
try:
# Solution from: https://www.tutorialspoint.com/How-to-convert-a-string-to-a-Python-class-object
architecture_object = getattr(sys.modules[__name__],
architecture)
rnn = RNN(architecture_object(units_layer), **args0)
except TypeError:
print('Unknown architecture.')
else:
architecture_object = getattr(sys.modules[__name__], architecture)
rnn = architecture_object(units_layer, **args0)
if mid_layer and novel_architecture:
model.add(Dropout(dropout))
model.add(rnn)
model.add(Dropout(dropout))
else:
model.add(rnn)
model.add(Dropout(dropout))
print('y_shape: ', y_shape, '\tX_shape: ', X_shape)
model.add(Dense(y_shape[1], activation='softmax'))
return model
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.activations import get as get_activation
from tensorflow.keras.layers import SimpleRNNCell, RNN, Layer
from tensorflow.keras.layers.experimental import LayerNormalization
class SimpleRNNCellWithLayerNorm(SimpleRNNCell):
def __init__(self, units, **kwargs):
self.activation = get_activation(kwargs.get("activation", "tanh"))
kwargs["activation"] = None
super().__init__(units, **kwargs)
self.layer_norm = LayerNormalization()
def call(self, inputs, states):
outputs, new_states = super().call(inputs, states)
norm_out = self.activation(self.layer_norm(outputs))
return norm_out, [norm_out]
model = Sequential([
RNN(SimpleRNNCellWithLayerNorm(20),
return_sequences=True,
input_shape=[None, 20]),
RNN(SimpleRNNCellWithLayerNorm(5)),
])
model.compile(loss="mse", optimizer="sgd")
X_train = np.random.randn(100, 50, 20)
Y_train = np.random.randn(100, 5)
history = model.fit(X_train, Y_train, epochs=2)
def build_wallscheid_lptn(x_shape, loss_weights=None,
verbose=True,
loss='mse',
# LPTN params
cap0=np.log10(1.0666e4), cap1=np.log10(6.5093e3),
cap2=np.log10(0.437127e3), cap3=np.log10(3.5105e3),
const_Rs0=np.log10(0.0375),
const_Rs1=np.log10(0.0707),
const_Rs2=np.log10(0.0899),
lin_Rs_slope=-54e-4,
lin_Rs_bias=np.log10(18e-3),
exp_Rs_magn0=1.7275, exp_Rs_magn1=0.8486,
exp_Rs_magn2=0.6349,
exp_Rs_b0=0.1573, exp_Rs_b1=0.1428, exp_Rs_b2=0.1184,
exp_Rs_a0=0.3039, exp_Rs_a1=0.2319, exp_Rs_a2=0.1205,
bipoly_Rs_magn=np.log10(0.3528),
bipoly_Rs_a=-0.2484,
bipoly_Rs_b=0.0276,
bipoly_Rs_c=0.3331,
ploss_Rdc=np.log10(14.6e-3),
ploss_alpha_cu=20e-4,
ploss_alpha_ac_1=0.562,
ploss_alpha_ac_2=0.2407,
ploss_beta_cu=2.5667,
ploss_k_1_0=0.5441, ploss_k_1_1=78e-4,
ploss_k_1_2=0.0352, ploss_k_1_3=-0.7438,
ploss_k_2=0.8655,
ploss_alpha_fe=-28e-4, schlepp_factor=1.4762,
):
"""O. Wallscheid, "Ein Beitrag zur thermischen Ausnutzung permanenterregter
Synchronmotoren in automobilen Traktionsanwendungen", Dissertation 2017
"""
x = Input(batch_input_shape=x_shape, ragged=False)
out = RNN(WallscheidLPTNCell(cap0, cap1, cap2, cap3,
const_Rs0, const_Rs1, const_Rs2,
lin_Rs_slope,
lin_Rs_bias,
exp_Rs_magn0, exp_Rs_magn1, exp_Rs_magn2,
exp_Rs_b0, exp_Rs_b1, exp_Rs_b2,
exp_Rs_a0, exp_Rs_a1, exp_Rs_a2,
bipoly_Rs_magn,
bipoly_Rs_a,
bipoly_Rs_b,
bipoly_Rs_c,
ploss_Rdc,
ploss_alpha_cu,
ploss_alpha_ac_1,
ploss_alpha_ac_2,
ploss_beta_cu,
ploss_k_1_0, ploss_k_1_1,
ploss_k_1_2, ploss_k_1_3,
ploss_k_2,
ploss_alpha_fe, schlepp_factor
),
return_sequences=True, stateful=True, name='rnn')(x)
if loss_weights is not None:
loss_weights = list(loss_weights)
model = Model(x, out)
model.compile(loss=loss, sample_weight_mode='temporal',
loss_weights=loss_weights, run_eagerly=False)
if verbose:
model.summary()
return model
output = prev_output + 1.
return output, [output]
batch_size = 3
time_step = 100 # interation
feature_size = 8
x0 = Input((feature_size))
layer0 = RepeatVector(time_step)
tmp = layer0(x0)
x1 = Input((feature_size))
cell = RNNCell(feature_size)
layer1 = RNN(cell)
y = layer1(inputs=tmp, initial_state=x1)
model = Model([x0, x1], y)
print(model.summary())
input0 = np.zeros((batch_size, feature_size), np.float32)
input1 = np.zeros((batch_size, feature_size), np.float32)
input1.fill(2.)
print(input0)
print(input1)
output0 = model.predict([input0, input1])
print(output0)
class LMU(Layer):
"""
A layer of trainable low-dimensional delay systems.
Each unit buffers its encoded input by internally representing a low-dimensional
(i.e., compressed) version of the input window.
Nonlinear decodings of this representation provide computations across the window,
such as its derivative, energy, median value, etc (*). Note that decoders can span
across all of the units.
By default the window lengths are trained via backpropagation, as well as the
encoding and decoding weights.
Optionally, the state-space matrices that implement the low-dimensional delay
system can be trained as well, but these are shared across all of the units in the
layer.
Based on the occurrence of the recurrent connections, this layer will choose
different implementations of evaluating the delay system.
If any recurrent connections are enabled, evaluation will occur sequentially with a
Keras RNN layer using the ``LMUCell`` cell class.
If all recurrent connections are disabled, evaluation of the delay system will be
computed as the convolution of the input sequence with the impulse response of
the LMU cell, using the ``LMUCellFFT`` cell class.
(*) Voelker and Eliasmith (2018). Improving spiking dynamical
networks: Accurate delays, higher-order synapses, and time cells.
Neural Computation, 30(3): 569-609.
(*) Voelker and Eliasmith. "Methods and systems for implementing
dynamic neural networks." U.S. Patent Application No. 15/243,223.
Filing date: 2016-08-22.
"""
def __init__(
self,
units,
order,
theta, # relative to dt=1
method="zoh",
realizer=Identity(), # TODO: Deprecate?
factory=LegendreDelay, # TODO: Deprecate?
memory_to_memory=True,
hidden_to_memory=True,
hidden_to_hidden=True,
trainable_input_encoders=True,
trainable_hidden_encoders=True,
trainable_memory_encoders=True,
trainable_input_kernel=True,
trainable_hidden_kernel=True,
trainable_memory_kernel=True,
trainable_A=False,
trainable_B=False,
input_encoders_initializer="lecun_uniform",
hidden_encoders_initializer="lecun_uniform",
memory_encoders_initializer=Constant(0), # 'lecun_uniform',
input_kernel_initializer="glorot_normal",
hidden_kernel_initializer="glorot_normal",
memory_kernel_initializer="glorot_normal",
hidden_activation="tanh",
return_sequences=False,
**kwargs):
# Note: Setting memory_to_memory, hidden_to_memory, and hidden_to_hidden to
# False doesn't actually remove the connections, but only initializes the
# weights to be zero and non-trainable (when using the LMUCell).
# This behaviour may change pending a future API decision.
self.units = units
self.order = order
self.theta = theta
self.method = method
self.realizer = realizer
self.factory = factory
self.memory_to_memory = memory_to_memory
self.hidden_to_memory = hidden_to_memory
self.hidden_to_hidden = hidden_to_hidden
self.trainable_input_encoders = trainable_input_encoders
self.trainable_hidden_encoders = (trainable_hidden_encoders
if hidden_to_memory else False)
self.trainable_memory_encoders = (trainable_memory_encoders
if memory_to_memory else False)
self.trainable_input_kernel = trainable_input_kernel
self.trainable_hidden_kernel = (trainable_hidden_kernel
if hidden_to_hidden else False)
self.trainable_memory_kernel = trainable_memory_kernel
self.trainable_A = trainable_A
self.trainable_B = trainable_B
self.input_encoders_initializer = input_encoders_initializer
self.hidden_encoders_initializer = (hidden_encoders_initializer if
hidden_to_memory else Constant(0))
self.memory_encoders_initializer = (memory_encoders_initializer if
memory_to_memory else Constant(0))
self.input_kernel_initializer = input_kernel_initializer
self.hidden_kernel_initializer = (hidden_kernel_initializer
if hidden_to_hidden else Constant(0))
self.memory_kernel_initializer = memory_kernel_initializer
self.hidden_activation = hidden_activation
self.return_sequences = return_sequences
super().__init__(**kwargs)
if self.fft_check():
self.lmu_layer = LMUCellFFT(
units=self.units,
order=self.order,
theta=self.theta,
trainable_input_encoders=self.trainable_input_encoders,
trainable_input_kernel=self.trainable_input_kernel,
trainable_memory_kernel=self.trainable_memory_kernel,
input_encoders_initializer=self.input_encoders_initializer,
input_kernel_initializer=self.input_kernel_initializer,
memory_kernel_initializer=self.memory_kernel_initializer,
hidden_activation=self.hidden_activation,
return_sequences=self.return_sequences,
)
else:
self.lmu_layer = RNN(
LMUCell(
units=self.units,
order=self.order,
theta=self.theta,
method=self.method,
realizer=self.realizer,
factory=self.factory,
trainable_input_encoders=self.trainable_input_encoders,
trainable_hidden_encoders=self.trainable_hidden_encoders,
trainable_memory_encoders=self.trainable_memory_encoders,
trainable_input_kernel=self.trainable_input_kernel,
trainable_hidden_kernel=self.trainable_hidden_kernel,
trainable_memory_kernel=self.trainable_memory_kernel,
trainable_A=self.trainable_A,
trainable_B=self.trainable_B,
input_encoders_initializer=self.input_encoders_initializer,
hidden_encoders_initializer=self.
hidden_encoders_initializer,
memory_encoders_initializer=self.
memory_encoders_initializer,
input_kernel_initializer=self.input_kernel_initializer,
hidden_kernel_initializer=self.hidden_kernel_initializer,
memory_kernel_initializer=self.memory_kernel_initializer,
hidden_activation=self.hidden_activation,
),
return_sequences=self.return_sequences,
)
def call(self, inputs):
"""
Calls the layer with inputs.
"""
return self.lmu_layer.call(inputs)
def build(self, input_shape):
"""
Initializes network parameters.
"""
self.lmu_layer.build(input_shape)
self.built = True
def fft_check(self):
"""
Checks if recurrent connections are enabled to
automatically switch to FFT.
"""
# Note: Only the flags are checked here. The alternative would be to check the
# weight initializers and trainiable flag settings, however it is cumbersome
# to check against all initializers forms that initialize the recurrent weights
# to 0.
#
# These flags used below exist in other LMUCell implementations, and will be
# brought forward in a future API decisions.
return not (self.memory_to_memory or self.hidden_to_memory
or self.hidden_to_hidden)
def get_config(self):
"""
Overrides the tensorflow get_config function.
"""
config = super().get_config()
config.update(
dict(
units=self.units,
order=self.order,
theta=self.theta,
method=self.method,
factory=self.factory,
memory_to_memory=self.memory_to_memory,
hidden_to_memory=self.hidden_to_memory,
hidden_to_hidden=self.hidden_to_hidden,
trainable_input_encoders=self.trainable_input_encoders,
trainable_hidden_encoders=self.trainable_hidden_encoders,
trainable_memory_encoders=self.trainable_memory_encoders,
trainable_input_kernel=self.trainable_input_kernel,
trainable_hidden_kernel=self.trainable_hidden_kernel,
trainable_memory_kernel=self.trainable_memory_kernel,
trainable_A=self.trainable_A,
trainable_B=self.trainable_B,
input_encorders_initializer=self.input_encoders_initializer,
hidden_encoders_initializer=self.hidden_encoders_initializer,
memory_encoders_initializer=self.memory_encoders_initializer,
input_kernel_initializer=self.input_kernel_initializer,
hidden_kernel_initializer=self.hidden_kernel_initializer,
memory_kernel_initializer=self.memory_kernel_initializer,
hidden_activation=self.hidden_activation,
return_sequences=self.return_sequences,
))
return config
def __init__(self, params):
cell = GRUCell(units=params['rnn_units'])
self.rnn = RNN(cell, return_state=True, return_sequences=True)
self.dense = Dense(units=2)
self.activate = Activate()
def __init__(
self,
units,
order,
theta, # relative to dt=1
method="zoh",
realizer=Identity(), # TODO: Deprecate?
factory=LegendreDelay, # TODO: Deprecate?
memory_to_memory=True,
hidden_to_memory=True,
hidden_to_hidden=True,
trainable_input_encoders=True,
trainable_hidden_encoders=True,
trainable_memory_encoders=True,
trainable_input_kernel=True,
trainable_hidden_kernel=True,
trainable_memory_kernel=True,
trainable_A=False,
trainable_B=False,
input_encoders_initializer="lecun_uniform",
hidden_encoders_initializer="lecun_uniform",
memory_encoders_initializer=Constant(0), # 'lecun_uniform',
input_kernel_initializer="glorot_normal",
hidden_kernel_initializer="glorot_normal",
memory_kernel_initializer="glorot_normal",
hidden_activation="tanh",
return_sequences=False,
**kwargs):
# Note: Setting memory_to_memory, hidden_to_memory, and hidden_to_hidden to
# False doesn't actually remove the connections, but only initializes the
# weights to be zero and non-trainable (when using the LMUCell).
# This behaviour may change pending a future API decision.
self.units = units
self.order = order
self.theta = theta
self.method = method
self.realizer = realizer
self.factory = factory
self.memory_to_memory = memory_to_memory
self.hidden_to_memory = hidden_to_memory
self.hidden_to_hidden = hidden_to_hidden
self.trainable_input_encoders = trainable_input_encoders
self.trainable_hidden_encoders = (trainable_hidden_encoders
if hidden_to_memory else False)
self.trainable_memory_encoders = (trainable_memory_encoders
if memory_to_memory else False)
self.trainable_input_kernel = trainable_input_kernel
self.trainable_hidden_kernel = (trainable_hidden_kernel
if hidden_to_hidden else False)
self.trainable_memory_kernel = trainable_memory_kernel
self.trainable_A = trainable_A
self.trainable_B = trainable_B
self.input_encoders_initializer = input_encoders_initializer
self.hidden_encoders_initializer = (hidden_encoders_initializer if
hidden_to_memory else Constant(0))
self.memory_encoders_initializer = (memory_encoders_initializer if
memory_to_memory else Constant(0))
self.input_kernel_initializer = input_kernel_initializer
self.hidden_kernel_initializer = (hidden_kernel_initializer
if hidden_to_hidden else Constant(0))
self.memory_kernel_initializer = memory_kernel_initializer
self.hidden_activation = hidden_activation
self.return_sequences = return_sequences
super().__init__(**kwargs)
if self.fft_check():
self.lmu_layer = LMUCellFFT(
units=self.units,
order=self.order,
theta=self.theta,
trainable_input_encoders=self.trainable_input_encoders,
trainable_input_kernel=self.trainable_input_kernel,
trainable_memory_kernel=self.trainable_memory_kernel,
input_encoders_initializer=self.input_encoders_initializer,
input_kernel_initializer=self.input_kernel_initializer,
memory_kernel_initializer=self.memory_kernel_initializer,
hidden_activation=self.hidden_activation,
return_sequences=self.return_sequences,
)
else:
self.lmu_layer = RNN(
LMUCell(
units=self.units,
order=self.order,
theta=self.theta,
method=self.method,
realizer=self.realizer,
factory=self.factory,
trainable_input_encoders=self.trainable_input_encoders,
trainable_hidden_encoders=self.trainable_hidden_encoders,
trainable_memory_encoders=self.trainable_memory_encoders,
trainable_input_kernel=self.trainable_input_kernel,
trainable_hidden_kernel=self.trainable_hidden_kernel,
trainable_memory_kernel=self.trainable_memory_kernel,
trainable_A=self.trainable_A,
trainable_B=self.trainable_B,
input_encoders_initializer=self.input_encoders_initializer,
hidden_encoders_initializer=self.
hidden_encoders_initializer,
memory_encoders_initializer=self.
memory_encoders_initializer,
input_kernel_initializer=self.input_kernel_initializer,
hidden_kernel_initializer=self.hidden_kernel_initializer,
memory_kernel_initializer=self.memory_kernel_initializer,
hidden_activation=self.hidden_activation,
),
return_sequences=self.return_sequences,
)
)
data_inputs.shape
expected_outputs.shape
sequence_length = data_inputs.shape[1]
input_dim = data_inputs.shape[2]
output_dim = expected_outputs.shape[2]
inputs = keras.Input(shape=(None, input_dim),
dtype=tf.dtypes.float32,
name="encoder_inputs")
encoder = RNN(
cell=[LayerNormLSTMCell(128),
LayerNormLSTMCell(128)],
return_sequences=True,
return_state=True,
)
encoder_output, *encoder_hidden = encoder(inputs)
context_vector, attention_weights = keras.layers.Attention()(
[encoder_hidden, encoder_output])
x = tf.expand_dims(context_vector, axis=1)
decoder = RNN(
cell=[LayerNormLSTMCell(128),
LayerNormLSTMCell(128)],
return_sequences=True,
return_state=False,
def train(type_name, n_hidden):
# Initialize save path
save_path = "/home/kevin/projects/exercise_pose_evaluation_machine/models/lstm_model/keras/" + type_name + "/" + type_name + "_lstm_model.h5"
# Get original dataset
x, y = get_dataset(type_name)
# Fill original class type with the label 1
y = [1 for label in y]
# Get negative dataset
neg_x, neg_y = get_dataset("not-" + type_name)
# Fill original class type with the label 1
neg_y = [0 for label in neg_y]
x.extend(neg_x)
y.extend(neg_y)
# Flatten X coodinates and filter
x = np.array(x)
_x = []
_y = []
for idx, data in enumerate(x):
data = [np.reshape(np.array(frames), (28)).tolist() for frames in data]
_x.append(data)
_y.append(y[idx])
x = _x
y = _y
# Split to training and test dataset
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
x_train = np.array(x_train)
x_test = np.array(x_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
# Define training parameters
n_classes = 1
# Make LSTM Layer
# Pair of lstm cell initialization through loop
# use_bias -> Adding bias vector on each layer, by default is True so this line could be deleted
# unit_forget_bias ->
lstm_cells = [
LSTMCell(n_hidden,
activation='relu',
use_bias=True,
unit_forget_bias=1.0) for _ in range(2)
]
stacked_lstm = StackedRNNCells(lstm_cells)
lstm_layer = RNN(stacked_lstm)
learning_rate = 1e-2
lr_schedule = PolynomialDecay(initial_learning_rate=learning_rate,
decay_steps=10,
end_learning_rate=0.00001)
optimizer = Adam(learning_rate=lr_schedule)
# Initiate model
# kernel_regularizers -> regularizing weights to avoid overfit training data on layer kernel
# activity_regularizer -> regularizing weights to avoid overfit training data on layer output
model = Sequential()
model.add(lstm_layer)
model.add(Dropout(0.3))
model.add(
Dense(n_classes,
activation='sigmoid',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01)))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# simple early stopping
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50)
# Train model
# shufffle = True -> shuffle training data
# validation_split -> portion of training data used for validation split
# validation_data -> external data used for validation
model.fit(x_train,
y_train,
epochs=450,
batch_size=150,
shuffle=True,
validation_data=(x_test, y_test),
validation_split=0.4,
callbacks=[es])
# Print model stats
print(model.summary())
print(model.get_config())
# Find accuracy
_, accuracy = model.evaluate(x_test, y_test)
print('Accuracy: %.2f' % (accuracy * 100))
# Save model
model.save(save_path)
print("Saved model!")
# Generate predictions
print("See prediction result")
random_int = random.randint(0, len(x_test))
data = x_test[random_int]
prediction = "1" if model.predict(np.array([data])) > 0.5 else "0"
print("predictions result:", prediction)
print("expected result: ", y_test[random_int])
def __init__(self, params):
self.params = params
self.rnn_cell = GRUCell(units=self.params['hidden_size'])
self.rnn = RNN(self.rnn_cell, return_state=True, return_sequences=True)
self.dense = Dense(units=self.params['inputdim'])
def __init__(self, num_classes, hparams):
super().__init__()
self.decoder_embedding = Embedding(input_dim=num_classes,
output_dim=hparams.embedding_dim)
self.rnn_cell = DecoderCell(num_classes, hparams)
self.rnn = RNN(self.rnn_cell, return_sequences=True, return_state=True)
def _init_model(self):
inputs = Input(shape=(self.input_dim_1, self.input_dim_2))
# No matter for LSTM, GRU, bidirection LSTM, final layer can not use 'return_sequences' output.
for i in range(self.n_layers - 1):
if i == 0:
res = self._rnn_block(inputs, name_index=i)
else:
res = self._rnn_block(res, name_index=i)
# final LSTM layer
if self.use_bidierc:
res = Bidirectional(LSTM(self.rnn_units))(res)
elif self.use_gru:
res = GRU(self.rnn_units)(res)
elif self.use_lstm:
res = LSTM(self.rnn_units)(res)
else:
res = RNN(self.rnn_units)(res)
# whether or not to use Dense layer
if self.use_dense:
res = Dense(self.dense_units)(res)
if self.use_batch:
res = BatchNormalization()(res)
res = Activation('relu')(res)
if self.use_dropout:
res = Dropout(self.drop_ratio)(res)
# this is method private function to check whether or not loss is not given, then use default loss
def _check_loss(model, loss, metrics, optimizer):
if loss is not None:
model.compile(loss=loss, metrics=[metrics], optimizer=optimizer)
return model
if self.n_classes == 2: # this is binary class problem.
out = Dense(self.n_classes, activation='sigmoid')(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='binary_crossentropy', metrics=[self.metrics], optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
elif self.n_classes >= 2: # this is multiclass problem.
out = Dense(self.n_classes, activation='softmax')(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='categorical_crossentropy', metrics=[self.metrics], optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
elif self.n_classes == -1: # this is regression problem
out = Dense(1)(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='mse', metrics=[self.metrics], optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
else:
raise AttributeError("Parameter 'n_classes' should be -1, 2 or up 2!")
if not self.silence:
print('Model structure summary:')
model.summary()
return model
def create_model(Bias_layer,
low,
up,
F,
beta,
gamma,
Co,
m,
a0RNN,
batch_input_shape,
selectbias,
selectidx,
selectdk,
selectwk,
myDtype,
return_sequences=False,
unroll=False):
batch_adjusted_shape = (batch_input_shape[2] + 1, ) #Adding state
placeHolder = Input(shape=(batch_input_shape[2] + 1, )) #Adding state
filterBias = inputsSelection(batch_adjusted_shape, selectbias)(placeHolder)
filterSig = inputsSelection(batch_adjusted_shape, selectidx)(placeHolder)
filterdK = inputsSelection(batch_adjusted_shape, selectdk)(placeHolder)
filterda = inputsSelection(batch_adjusted_shape, selectwk)(placeHolder)
MLP_min = low
MLP_range = up - low
Bias_layer = Bias_layer(filterBias)
MLP = Lambda(lambda x: ((x * MLP_range) + MLP_min))(Bias_layer)
Filter = Lambda(lambda x: sign(x))(filterSig)
Bias_filtered_layer = Multiply()([MLP, Filter])
dk_input_shape = filterdK.get_shape()
dkLayer = StressIntensityRange(input_shape=dk_input_shape,
dtype=myDtype,
trainable=False)
dkLayer.build(input_shape=dk_input_shape)
dkLayer.set_weights([np.asarray([F], dtype=dkLayer.dtype)])
dkLayer = dkLayer(filterdK)
wmInput = Concatenate(axis=-1)([dkLayer, filterda])
wm_input_shape = wmInput.get_shape()
wmLayer = WalkerModel(input_shape=wm_input_shape,
dtype=myDtype,
trainable=False)
wmLayer.build(input_shape=wm_input_shape)
wmLayer.set_weights(
[np.asarray([beta, gamma, Co, m], dtype=wmLayer.dtype)])
wmLayer = wmLayer(wmInput)
da_layer = Add()([Bias_filtered_layer, wmLayer])
functionalModel = Model(inputs=[placeHolder], outputs=[da_layer])
"-------------------------------------------------------------------------"
CDMCellHybrid = CumulativeDamageCell(model=functionalModel,
batch_input_shape=batch_input_shape,
dtype=myDtype,
initial_damage=a0RNN)
CDMRNNhybrid = RNN(cell=CDMCellHybrid,
return_sequences=return_sequences,
return_state=False,
batch_input_shape=batch_input_shape,
unroll=unroll)
model = Sequential()
model.add(CDMRNNhybrid)
model.compile(loss=mape, optimizer=RMSprop(1e-11), metrics=['mse'])
return model
np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] # build model inputs = Input(shape=(doc_length, )) embed = Embedding(vocab_size, hidden_size, input_length=doc_length) embed_input = embed(inputs) #lstm=LSTM(hidden_size, return_sequences=True)(embed_input) cell = PeepholeLSTMCell(hidden_size, implementation=1) lstm = Bidirectional(RNN(cell, return_sequences=True))(embed_input) dropout = Dropout(0.5)(lstm) meanpool = Lambda(lambda x: mean(x, axis=1))(dropout) hidden = Dense(100)(meanpool) output = Dense(1, activation='sigmoid')(hidden) model = Model(inputs=inputs, outputs=output) opt = Adam() model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy']) print(model.summary()) model.fit(x_train, y_train, epochs=epoch, verbose=1) # evaluate the model loss, accuracy = model.evaluate(x_dev, y_dev, verbose=0)
X = X_train[idx].reshape(1, 1)
Y = Y_train[idx].reshape(1, NUM_WORDS)
# pdb.set_trace()
yield X, Y
sentences_as_index = convert_sentences_idx(sentences, word_to_idx)
train_generator, valid_generator = create_train_valid_sentences(
sentences_as_index, NUM_WORDS)
pdb.set_trace()
# setup the RNN
cells = [MinimalRNNCell(CELL_OUTPUTS, EMBED_SIZE, NUM_WORDS)]
rnn = RNN(cells, return_sequences=False, input_shape=(None, 1))
model = Sequential()
model.add(
Embedding(input_dim=NUM_WORDS,
output_dim=EMBED_SIZE,
weights=[embedding_matrix],
trainable=False))
model.add(rnn)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
checkpoint_path = "checkpoints/cp-{epoch:04d}.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
def build_ncp_model(input_shape, config, stateful=False, batch_size=1, draw_ncp_internals=False):
## Define NCP
ncp_wiring = wirings.NCP(
inter_neurons=config["inter_neurons"], # Number of inter neurons
command_neurons=12, # Number of command neurons
motor_neurons=1, # Number of motor neurons
sensory_fanout=6, # How many outgoing synapses has each sensory neuron
inter_fanout=config["inter_fanout"], # How many outgoing synapses has each inter neuron
recurrent_command_synapses=config["recurrent_command_synapses"], # Now many recurrent synapses are in the command neuron layer
motor_fanin=8, # How many incomming syanpses has each motor neuron
)
ncp_cell = LTCCell(ncp_wiring)
## Build model ##
# INPUT_SHAPE = (timesteps, *input_shape)
if not stateful:
batch_size = None
IMAGE_INPUT_SHAPE = (batch_size, None, *input_shape)
STEER_INPUT_SHAPE = (batch_size, None, config["command_input_size"])
# Create CNN part of the network based on Nvidia model
model_cnn = Sequential()
model_cnn.add(Lambda(lambda x: x / 127.5 - 1.0, batch_input_shape=IMAGE_INPUT_SHAPE, name="image_normalization"))
model_cnn.add(TimeDistributed(Conv2D(24, 5, activation='elu', strides=(2, 2)), name="convolution_1"))
model_cnn.add(TimeDistributed(Conv2D(36, 5, activation='elu', strides=(2, 2)), name="convolution_2"))
model_cnn.add(TimeDistributed(Conv2D(48, 5, activation='elu', strides=(2, 2)), name="convolution_3"))
model_cnn.add(TimeDistributed(Conv2D(64, 3, activation='elu'), name="convolution_4"))
model_cnn.add(TimeDistributed(Conv2D(16, 3, activation='elu'), name="convolution_5"))
model_cnn.add(TimeDistributed(Dropout(0.3), name="convolution_dropout"))
model_cnn.add(TimeDistributed(Flatten(), name="convolution_flatten"))
model_cnn.add(TimeDistributed(Dense(100, activation='elu'), name="convolution_inner_dense"))
model_cnn.add(TimeDistributed(Dense(config["convolution_out_size"], activation='elu'), name="convolution_output_dense"))
# Create input for Steering data
steer_input = Input(batch_input_shape=STEER_INPUT_SHAPE, name="command_input")
model_steer = steer_input
if config["use_dense_size"]:
model_steer = TimeDistributed(Dense(config["use_dense_size"], activation="elu"), name="command_dense")(steer_input)
model_steer = TimeDistributed(Flatten(), name="command_output_flatten")(model_steer)
model_steer = Model(inputs=steer_input, outputs=model_steer)
# Combine CNN part with Steering input
combined = concatenate([model_cnn.output, model_steer.output], name="combine_cnn_command")
# Add Recurrent NCP layer to the model
z = RNN(ncp_cell, return_sequences=False, stateful=stateful, name="NCP_RNN")(combined)
z = Activation("tanh", name="tanh_output_activation")(z)
# model.add(TimeDistributed(Dense(1)))
# Finish model
model = Model(inputs=[model_cnn.inputs, model_steer.inputs], outputs=z)
# Visualize NCP connections - must be at the end after model is created
if draw_ncp_internals:
sns.set_style("white")
plt.figure(figsize=(12, 8))
legend_handles = ncp_cell.draw_graph(neuron_colors={"command": "tab:cyan"})
plt.legend(handles=legend_handles, loc="upper center", bbox_to_anchor=(1.1, 1.1))
sns.despine(left=True, bottom=True)
plt.tight_layout()
plt.show()
return model
def __init__(self, params):
self.params = params
cell = GRUCell(units=self.params['rnn_size'])
self.rnn = RNN(cell, return_state=True, return_sequences=True)
self.dense = Dense(units=1)
def nBRU(hidden_size, return_sequences=False, **kwargs):
return RNN(nBRUCell(hidden_size, **kwargs), return_sequences)
def _create_model_predictor(self, lr):
y = self.y.ravel()
# Initial values from statsmodels implementation
if self.seasonal:
l0_start = y[np.arange(y.size) % self.seasonal_period == 0].mean()
lead, lag = y[self.seasonal_period: 2 * self.seasonal_period], y[:self.seasonal_period]
if self.trend == "multiplicative":
b0_start = ((lead - lag) / self.seasonal_period).mean()
elif self.trend == "additive":
b0_start = np.exp((np.log(lead.mean()) - np.log(lag.mean())) / self.seasonal_period)
if self.seasonal == "multiplicative":
s0_start = y[:self.seasonal_period] / l0_start
elif self.seasonal == "additive":
s0_start = y[:self.seasonal_period] - l0_start
elif self.trend:
l0_start = y[0]
if self.trend == "multiplicative":
b0_start = y[1] / l0_start
elif self.trend == "additive":
b0_start = y[1] - l0_start
else:
l0_start = y[0]
inp_y = Input(shape=(None, 1))
inp_emb_id = Input(shape=(1,)) # Dummy ID for embeddings
# (Ab)using embeddings here for initial value variables
init_l0 = [Embedding(1, 1, embeddings_initializer=constant(l0_start), name="l0")(inp_emb_id)[:, 0, :]]
if self.trend:
init_b0 = [Embedding(1, 1, embeddings_initializer=constant(b0_start), name="b0")(inp_emb_id)[:, 0, :]]
else:
init_b0 = []
if self.seasonal:
init_seas_emb = Embedding(1, self.seasonal_period, embeddings_initializer=RandomUniform(0.8, 1.2),
name="s0")
init_seas = [init_seas_emb(inp_emb_id)[:, 0, :]]
else:
init_seas = []
rnncell = ESRNN(self.trend, self.seasonal, self.seasonal_period)
rnn = RNN(rnncell, return_sequences=True, return_state=True)
out_rnn = rnn(inp_y, initial_state=init_l0 + init_b0 + init_seas)
if self.trend == "multiplicative":
l0_b0 = init_l0[0] * init_b0[0]
elif self.trend == "additive":
l0_b0 = init_l0[0] + init_b0[0]
else:
l0_b0 = init_l0[0]
if self.seasonal == "multiplicative":
l0_b0_s0 = l0_b0 * init_seas[0][:, :1]
elif self.seasonal == "additive":
l0_b0_s0 = l0_b0 + init_seas[0][:, :1]
else:
l0_b0_s0 = l0_b0
out = tf.keras.layers.concatenate([
tf.math.reduce_sum(l0_b0_s0, axis=1)[:, None, None],
out_rnn[0]
], axis=1)
model = Model(inputs=[inp_y, inp_emb_id], outputs=out)
model.compile(Adam(lr), "mse")
# predictor also outputs final state for predicting out-of-sample
predictor = Model(inputs=[inp_y, inp_emb_id], outputs=[out, out_rnn[1:]])
# Assign initial seasonality weights
if self.seasonal:
init_seas_emb.set_weights([s0_start.reshape(init_seas_emb.get_weights()[0].shape)])
return model, predictor
def create_model(MLP_C_layer,
MLP_m_layer,
low_C,
up_C,
low_m,
up_m,
F,
a0RNN,
batch_input_shape,
selectaux,
selectdk,
myDtype,
return_sequences=False,
unroll=False):
batch_adjusted_shape = (batch_input_shape[2] + 1, ) #Adding state
placeHolder = Input(shape=(batch_input_shape[2] + 1, )) #Adding state
filterLayer = inputsSelection(batch_adjusted_shape, selectaux)(placeHolder)
filterdkLayer = inputsSelection(batch_adjusted_shape,
selectdk)(placeHolder)
MLP_C_min = low_C
MLP_C_range = up_C - low_C
MLP_C_layer = MLP_C_layer(filterLayer)
C_layer = Lambda(lambda x: ((x * MLP_C_range) + MLP_C_min))(MLP_C_layer)
MLP_m_min = low_m
MLP_m_range = up_m - low_m
MLP_m_layer = MLP_m_layer(filterLayer)
MLP_scaled_m_layer = Lambda(lambda x: ((x * MLP_m_range) + MLP_m_min))(
MLP_m_layer)
dk_input_shape = filterdkLayer.get_shape()
dkLayer = StressIntensityRange(input_shape=dk_input_shape,
dtype=myDtype,
trainable=False)
dkLayer.build(input_shape=dk_input_shape)
dkLayer.set_weights([np.asarray([F], dtype=dkLayer.dtype)])
dkLayer = dkLayer(filterdkLayer)
ldK_layer = Lambda(lambda x: tf.math.log(x) /
(tf.math.log(tf.constant(10.))))(dkLayer)
dKm_layer = Multiply()([MLP_scaled_m_layer, ldK_layer])
aux_layer = Add()([C_layer, dKm_layer])
da_layer = Lambda(lambda x: 10**(x))(aux_layer)
functionalModel = Model(inputs=[placeHolder], outputs=[da_layer])
"-------------------------------------------------------------------------"
CDMCellHybrid = CumulativeDamageCell(model=functionalModel,
batch_input_shape=batch_input_shape,
dtype=myDtype,
initial_damage=a0RNN)
CDMRNNhybrid = RNN(cell=CDMCellHybrid,
return_sequences=return_sequences,
return_state=False,
batch_input_shape=batch_input_shape,
unroll=unroll)
model = Sequential()
model.add(CDMRNNhybrid)
model.compile(loss='mse',
optimizer=RMSprop(learning_rate=1e-6),
metrics=['mae'])
return model
