def OptimTypeFunc(x, OpLr, OpMom):
# Function to determine to get desiered optimezer
return {
0: Nadam(lr=OpLr),
1: Adamax(lr=OpLr),
2: SGD(lr=OpLr, momentum=OpMom),
3: Adam(lr=OpLr, clipnorm=1.0),
4: Nadam(lr=OpLr)
}[x]
def __init__(
self,
input_shape,
number_of_classes,
filtres=16,
tailleBlock={
'A': 10,
'B': 3,
'C': 3
},
optimiseur='Nadam',
activation='elu',
beta=1.1,
initializer='he_normal',
metrics=['accuracy'],
learningR=None, #0.0005,
nb_gpu=2):
get_custom_objects()['swish'] = swish
get_custom_objects()['e_swish'] = e_swish
self.input_shape = input_shape
self.number_of_classes = number_of_classes
self.filtres = filtres
self.tailleBlock = tailleBlock
#if learningR is not None :
self.optimiseur = optimiseur
if learningR is not None:
self.optimiseur = {
'SGD': SGD(learning_rate=learningR),
'RMSprop': RMSprop(learning_rate=learningR),
'Adagrad': Adagrad(learning_rate=learningR),
'Adadelta': Adadelta(learning_rate=learningR),
'Adam': Adam(learning_rate=learningR),
'Adamax': Adamax(learning_rate=learningR),
'Nadam': Nadam(learning_rate=learningR),
}[optimiseur]
else:
self.optimiseur = {
'SGD': SGD(),
'RMSprop': RMSprop(),
'Adagrad': Adagrad(),
'Adadelta': Adadelta(),
'Adam': Adam(),
'Adamax': Adamax(),
'Nadam': Nadam(),
}[optimiseur]
self.activation = activation
self.initializer = initializer
self.nb_gpu = nb_gpu
self.metrics = metrics
# la valeur 3 indique que les canaux des couleurs sont à la fin
# autrement -1 (je n'utilise pas cette syntaxe )
self.channel_axis = 3
def __init__(self,
update_after='each',
epsilon=0.85,
EPS_DECAY=0.998,
OBSERVATION_SPACE_SIZE=(5, 5, 3),
ACTION_SPACE_SIZE=4,
ep_number=0,
RENDER_EVERY=200,
DISCOUNT=0.50,
BATCH_SIZE=350):
#Update after: All = all subtasks; Each = each subtask
self.update_after = update_after
self.OBSERVATION_SPACE_SIZE = OBSERVATION_SPACE_SIZE
self.ACTION_SPACE_SIZE = ACTION_SPACE_SIZE
self.ep_number = ep_number
self.RENDER_EVERY = RENDER_EVERY # render every 1000 episodes
self.DISCOUNT = DISCOUNT
self.BATCH_SIZE = BATCH_SIZE
self.epsilon = epsilon
self.EPSILON_DECAY = EPS_DECAY
self.MIN_EPSILON = 0.05
self.logdir = "logs"
self.REPLAY_BUFFER_SIZE = 10 * self.BATCH_SIZE
self.epoch_history = deque(maxlen=self.REPLAY_BUFFER_SIZE)
self.REWARDS = []
self.MEAN_REWARDS = []
self.WINDOW_SIZE = 100
self.model = StackCNNDenseModel(
input_dim=self.OBSERVATION_SPACE_SIZE,
output_dim=self.ACTION_SPACE_SIZE,
hidden_dim=4,
num_hidden_layers=2,
activation='linear',
initializer=tf.keras.initializers.glorot_normal()
) # self.create_model_simple() # tf.keras.models.load_model('model')
self.model.compile(loss="mse",
optimizer=Nadam(lr=0.005),
metrics=['mae'])
self.target_model = StackCNNDenseModel(
input_dim=self.OBSERVATION_SPACE_SIZE,
output_dim=self.ACTION_SPACE_SIZE,
hidden_dim=4,
num_hidden_layers=2,
activation='linear',
initializer=tf.keras.initializers.glorot_normal())
self.target_model.compile(loss="mse",
optimizer=Nadam(lr=0.005),
metrics=['mae'])
def train_model(df, days_ahead, num_iter):
print("Training {}...".format(days_ahead))
for target in targets:
df["target_{}".format(target)] = df.groupby("loc")[target].shift(1-days_ahead)
train_df = df[df["target_{}".format(targets[0])].notnull()].copy()
X_hist_train = np.zeros((train_df.shape[0], NUM_SHIFT, len(targets)))
for i in range(len(targets)):
X_hist_train[:, :, i] = train_df[ts_features[targets[i]]].values
epochs = [10, 4]
if days_ahead >= 20:
epochs = [6, 2]
for it in range(num_iter):
print("Iteration:", it)
K.clear_session()
model = get_model()
for loss, lr, epoch in [(get_custom_loss([0.15, 0.50, 0.85]), 0.0001, epochs[0]), (get_custom_loss([0.05, 0.50, 0.95]), 0.00005, epochs[1])]:
model.compile(loss=loss, optimizer=SWA(Nadam(lr=lr), average_period=2))
model.fit(X_hist_train, get_output(train_df), sample_weight=get_weight(train_df),
shuffle=True, batch_size=128, epochs=epoch, verbose=0)
model.save_weights("weights/model_{d}_{it}.h5".format(d=days_ahead, it=it))
def construct_model():
"""
Construct the CNN model.
***
Please add your model implementation here, and don't forget compile the model
E.g., model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
NOTE, You must include 'accuracy' in as one of your metrics, which will be used for marking later.
***
:return: model: the initial CNN model
"""
model = Sequential([
Conv2D(16, kernel_size=3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)),
MaxPooling2D(2, 2),
Conv2D(32, kernel_size=3, padding='same', activation='relu'),
MaxPooling2D(2, 2),
Conv2D(64, 3, padding='same', activation='relu'),
MaxPooling2D(2, 2),
Conv2D(128, 3, padding='same', activation='relu'),
Flatten(),
Dense(64, 'relu'),
Dropout(0.5),
Dense(3, activation='softmax')
])
model.compile(loss=categorical_crossentropy, optimizer= Nadam(), metrics=['accuracy'])
return model
def speed_model():
inputs = Input(shape=IMG_SHAPE)
inputs1 = Lambda(lambda x: x/ 127.5 - 1, input_shape = IMG_SHAPE)(inputs)
conv1 = Conv2D(24, (5, 5), padding="valid")(inputs1)
act1 = Activation(ELU())(conv1)
conv2 = Conv2D(36, (5, 5), strides=(2, 2), padding="valid")(act1)
act2 = Activation(ELU())(conv2)
drop1 = Dropout(0.5)(act2)
conv3 = Conv2D(48, (5, 5), strides=(2, 2), padding="valid")(drop1)
act3 = Activation(ELU())(conv3)
conv4 = Conv2D(64, (3, 3), strides=(1, 1), padding="valid")(act3)
act4 = Activation(ELU())(conv4)
conv5 = Conv2D(64, (3, 3), padding="valid")(act4)
flat1 = Flatten()(conv5)
act4 = Activation(ELU())(flat1)
dense1 = Dense(100)(act4)
act5 = Activation(ELU())(dense1)
dense2 = Dense(50)(act5)
act6 = Activation(ELU())(dense2)
dense4 = Dense(10)(act6)
act8 = Activation(ELU())(dense4)
output = Dense(1)(act8)
model = Model(inputs, output)
adam = Nadam()
model.compile(optimizer=adam, loss='mse')
print(model.summary())
return model
def _build_model(self) -> Sequential:
model = Sequential()
for state in self.sequence:
if isinstance(state, Input):
model.add(layers.Input(shape=(state.units,)))
elif isinstance(state, Hidden):
model.add(layers.Dense(units=state.units, activation=state.activation))
elif isinstance(state, Dropout):
model.add(layers.Dropout(rate=state.rate))
elif isinstance(state, Output):
model.add(layers.Dense(units=state.units, activation=state.activation))
optimizer = OPTIMIZER
if optimizer == 'adam':
optimizer = Adam(LEARNING_RATE)
elif optimizer == 'nadam':
optimizer = Nadam(LEARNING_RATE)
elif optimizer == 'rmsprop':
optimizer = RMSprop(LEARNING_RATE)
metrics = []
for metric in METRICS:
if metric == 'r_square':
metric = RSquare(name=metric, y_shape=(1,))
elif metric == 'rmse':
metric = RootMeanSquaredError(metric)
metrics.append(metric)
model.compile(
optimizer=optimizer,
loss=LOSS,
metrics=metrics
)
return model
def evaluate(self, device: str, strategy_name, pretrained):
# init strategy
if strategy_name == 'tpu':
resolver = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
print('TPU strategy selected.')
else:
strategy = tf.distribute.OneDeviceStrategy(device=device)
print('GPU strategy selected.')
dl = DataLoader(self.record_paths)
self.test_ds = dl.get_dataset(
'test', batch_size=self.config['train_batch_size'])
# infer steps for fitting and data structure
test_steps = get_dataset_size(self.test_ds)
data_structure = get_dataset_structure(self.test_ds)
with strategy.scope():
model = get_model(input_shape=data_structure, name="VGG16_tf")
model.load_weights(
os.path.join(MODEL_PATH, pretrained, 'checkpoint'))
optimizer = Nadam()
model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
optimizer=optimizer,
metrics=[
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.Recall(),
tf.keras.metrics.Precision()
])
model.summary()
model.evaluate(self.test_ds, steps=test_steps)
def create(self, input_shape, segment_length, step, optimizer, learning_rate, verbose):
identifier = f'Conv1D_pred'
super().__init__(input_shape, segment_length, step,
n_output_classes=1,
loss_function='mean_squared_error',
metrics=['mse'],
identifier=identifier,
verbose=verbose)
self.add(Reshape((segment_length, 1), input_shape=(input_shape,)))
self.add(Conv1D(128, 2, activation='relu', input_shape=(segment_length, 1)))
self.add(MaxPooling1D(pool_size=2, strides=1))
self.add(Conv1D(64, 2, activation='relu'))
self.add(GlobalAveragePooling1D())
self.add(Flatten())
self.add(Dense(10, activation='relu'))
self.add(Dense(1, activation='linear'))
if optimizer == 'sgd':
if learning_rate:
self.optimizer = SGD(lr=learning_rate, nesterov=True)
else:
self.optimizer = SGD(nesterov=True)
elif optimizer == 'nadam' and learning_rate:
self.optimizer = Nadam(lr=learning_rate)
self.compile()
def read_model(file):
model = get_unet0()
model.compile(optimizer=Nadam(lr=1e-3),
loss=jaccard_coef_loss,
metrics=['binary_crossentropy', jaccard_coef_int])
model.load_weights(file)
return model
def deep_lstm_model(image_shape, sequence_length, classes):
(h, w, c) = image_shape
sequence_shape = (sequence_length, h, w, c)
video = Input(shape=sequence_shape)
# cnn_base = VGG16(input_shape=image_shape,
# weights="imagenet",
# include_top=False)
extractor_base = Xception(
include_top=False,
weights='imagenet',
input_shape=image_shape,
pooling='avg',
)
# cnn_out = GlobalAveragePooling2D()(cnn_base.output)
extractor = Model(extractor_base.input, extractor_base.output)
extractor.trainable = False
encoded_frames = TimeDistributed(extractor)(video)
encoded_sequences = LSTM(64, return_sequences=True)(encoded_frames)
encoded_sequence = LSTM(64)(encoded_sequences)
hidden_layer = Dense(256, activation="relu")(encoded_sequence)
outputs = Dense(classes, activation="softmax")(hidden_layer)
model = Model(video, outputs)
optimizer = Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004)
model.compile(loss="sparse_categorical_crossentropy",
optimizer=optimizer,
metrics=["categorical_accuracy"])
return model
def build(self):
"""Builds the network using Keras.
"""
word_inputs = Input(shape=(None, MAX_WORD_LENGTH + 2), dtype="int32")
inputs = [word_inputs]
word_outputs = self._build_word_cnn(word_inputs)
if len(self.word_vectorizers) > 0:
additional_word_inputs = [
Input(shape=(None, input_dim), dtype="float32")
for input_dim, dense_dim in self.word_vectorizers
]
inputs.extend(additional_word_inputs)
additional_word_embeddings = [
Dense(dense_dim)(additional_word_inputs[i])
for i, (_, dense_dim) in enumerate(self.word_vectorizers)
]
word_outputs = Concatenate()([word_outputs] +
additional_word_embeddings)
outputs, lstm_outputs = self._build_basic_network(word_outputs)
compile_args = {
"optimizer": Nadam(lr=0.002, clipnorm=5.0),
"loss": "categorical_crossentropy",
"metrics": ["accuracy"]
}
self.model_ = Model(inputs, outputs)
self.model_.compile(**compile_args)
if self.verbose > 0:
self.model_.summary(print_fn=log.info)
return self
def fit_fnn(x_train, y_train, x_val):
tf.keras.backend.clear_session()
es = EarlyStopping(monitor='val_loss', mode='min', verbose=2, patience=2)
X_train_array = x_train.values.reshape(
(x_train.shape[0], x_train.shape[1], 1))
y_val = x_val[['X(t)']]
X_val_array = x_val.drop(['X(t)'], axis=1).values.reshape(
(x_val.shape[0], x_val.drop(['X(t)'], axis=1).shape[1], 1))
model = Sequential()
model.add(
Dense(X_train_array.shape[1],
activation='relu',
input_shape=(X_train_array.shape[1], X_train_array.shape[2])))
model.add(Dense(X_train_array.shape[1], activation='relu'))
model.add(Dense(X_train_array.shape[1], activation='relu'))
model.add(Dense(X_train_array.shape[1], activation='relu'))
model.add(Dense(X_train_array.shape[1], activation='relu'))
model.add(Flatten())
model.add(Dense(X_train_array.shape[1] // 2, activation='relu'))
model.add(Dense(1))
nadam = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999)
model.compile(optimizer=nadam, loss='mae', metrics=['mse'])
# fit network
model.fit(X_train_array,
y_train,
epochs=100,
batch_size=128,
validation_data=(X_val_array, y_val),
shuffle=False,
verbose=2,
callbacks=[es])
return model
def learn_model(log, attributes, epochs, early_stop):
num_activities = len(log.values[log.activity]) + 1
# Input + Embedding layer for every attribute
input_layers = []
embedding_layers = []
for attr in attributes:
if attr not in log.ignoreHistoryAttributes and attr != log.time and attr != log.trace:
for k in range(log.k):
i = Input(shape=(1, ),
name=attr.replace(" ", "_").replace("(", "").replace(
")", "").replace(":", "_") + "_Prev%i" % k)
input_layers.append(i)
# e = Embedding(len(log.values[attr]) + 1, 32, embeddings_initializer="zeros")(i)
e = Embedding(len(log.values[attr]) + 1,
len(log.values[attr]) + 1,
embeddings_initializer="zeros")(i)
embedding_layers.append(e)
concat = Concatenate()(embedding_layers)
drop = Dropout(0.2)(concat)
dense2 = Dense(num_activities)(drop)
flat = Flatten()(dense2)
output = Softmax(name="output")(flat)
model = Model(inputs=input_layers, outputs=[output])
opt = Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004,
clipvalue=3)
model.compile(loss={'output': 'categorical_crossentropy'}, optimizer=opt)
model.summary()
early_stopping = EarlyStopping(monitor='val_loss', patience=early_stop)
outfile = 'tmp/model_{epoch:03d}-{val_loss:.2f}.h5'
model_checkpoint = ModelCheckpoint(outfile,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto')
x, y, vals = transform_data(
log, [a for a in attributes if a != log.time and a != log.trace])
if len(y) < 10:
split = 0
else:
split = 0.2
model.fit(x=x,
y=y,
validation_split=split,
verbose=2,
callbacks=[early_stopping],
batch_size=32,
epochs=epochs)
return model
def create_model():
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
shape = (48, 96, 1)
filters = 32
depth = 8
stacks = 1
input_img = Input(shape)
model = MaskedConvolution2D(filters=filters, kernel_size=(7, 7), padding='same', mask='A')(input_img)
# model = InitialBlock(filters)(input_img)
model = ResidualBlockList(filters, depth, stacks)(model)
for _ in range(2):
model = Convolution2D(filters, (1, 1), padding='valid')(model)
model = ReLU()(model)
outs = Convolution2D(1, (1, 1), padding='valid')(model)
outs = Activation('sigmoid')(outs)
model = Model(input_img, outs)
model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
return model
def newCNN_model_new(num_variables, optimizer, nClasses, nl, dp):
model = Sequential()
model.add(
Dense(36,
input_dim=num_variables,
kernel_initializer='glorot_normal',
activation='relu'))
model.add(Reshape((1, 6, 6), input_shape=(36, )))
#model.add(Conv2D(256, (3, 3), padding='same',activation='relu'))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(16, (3, 3), padding='same', activation='relu'))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(25, activation='relu'))
model.add(Dense(nClasses, activation='softmax'))
if optimizer == 'Adam':
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.01),
metrics=['acc'])
if optimizer == 'Nadam':
model.compile(loss='categorical_crossentropy',
optimizer=Nadam(lr=0.01),
metrics=['acc'])
if optimizer == 'RMSprop':
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(learning_rate=0.001),
metrics=['acc'])
return model
def buildModel():
print("Building Convolutional Neural Network")
model = Sequential()
model.add(Conv2D(32, (8, 8), padding="same", strides=(4, 4), input_shape=(
img_cols, img_rows, img_channels))) # First layer of 80*80*4 with 32 filters
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Activation("relu"))
# Second layer of 40*40*4 with 64 filters
model.add(Conv2D(64, (4, 4), strides=(2, 2), padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Activation("relu"))
# Third layer of 30*30*4 with 64 filters
model.add(Conv2D(64, (3, 3), strides=(1, 1), padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Activation("relu"))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation("relu"))
model.add(Dense(ACTIONS))
#adam = Adam(lr=LEARNING_RATE)
nadam = Nadam(lr=LEARNING_RATE)
model.compile(loss="mse", optimizer=nadam)
# Creating model file if not present
if not os.path.isfile(LOSS_FILE_PATH):
model.save_weights("model.h5")
print("Finished building the Convolutional Neural Network")
return model
def init_model(self):
self.model = Sequential()
self.model.add(
Dense(self.hidden_units,
input_dim=self.input_units,
activation=self.activation))
self.model.add(
Dropout(self.dropout, noise_shape=self.noise_shape,
seed=self.seed))
self.model.add(
Dense(self.output_units, activation=self.activation_last))
if self.optimizer == 'RMSprop':
opt = RMSprop(learning_rate=self.learning_rate)
elif self.optimizer == 'Adadelta':
opt = kAdadelta(learning_rate=self.learning_rate)
elif self.optimizer == 'SGD':
opt = SGD(learning_rate=self.learning_rate)
elif self.optimizer == 'Adagrad':
opt = Adagrad(learning_rate=self.learning_rate)
elif self.optimizer == 'Adamax':
opt = Adamax(learning_rate=self.learning_rate)
elif self.optimizer == 'Nadam':
opt = Nadam(learning_rate=self.learning_rate)
else:
opt = Adam(learning_rate=self.learning_rate)
self.model.compile(optimizer=opt,
loss=self.loss,
metrics=[self.metrics])
def create_model():
# import Inception v3 model trained on imagenet
base_model = tf.keras.applications.InceptionV3(
input_shape=(96, 96, 3),
include_top=False,
weights="imagenet")
base_model.trainable = False
# initialize top layers
model = Sequential([
base_model,
GlobalAveragePooling2D(),
Dropout(0.3),
Dense(4, activation='softmax')
])
# compile model
model.compile(
optimizer=Nadam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07),
loss="categorical_crossentropy",
metrics=["accuracy"])
return model
def fit(self, X_train, y_train):
# X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train)
input_size = X_train.shape[1]
if self.type == 'custom':
self.make_custom_model(input_size)
elif self.type == 'larger':
self.larger_model(input_size)
elif self.type == 'wider':
self.wider_model(input_size)
print(self.model.summary())
# model_json = self.model.to_json()
# with open("DL_model.json", "w") as json_file:
# json_file.write(model_json)
self.model.compile(loss="mean_squared_error", optimizer=Nadam(lr=1e-4))
hist = self.model.fit(X_train,
y_train,
batch_size=5,
epochs=500,
verbose=0,
shuffle=True)
# KerasRegressor(build_fn=larger_model, epochs=50, batch_size=5, verbose=0))
return self.model
def nvidia_model():
model = Sequential()
# normalization
# perform custom normalization before lambda layer in network
model.add(Lambda(lambda x: x/ 127.5 - 1, input_shape = IMG_SHAPE))
model.add(Conv2D(24, (5, 5),
strides=(2,2),
padding = 'valid',
kernel_initializer = 'he_normal',
name = 'conv1'))
model.add(ELU())
model.add(Conv2D(36, (5, 5),
strides=(2,2),
padding = 'valid',
kernel_initializer = 'he_normal',
name = 'conv2'))
model.add(ELU())
model.add(Conv2D(48, (5, 5),
strides=(2,2),
padding = 'valid',
kernel_initializer = 'he_normal',
name = 'conv3'))
model.add(ELU())
model.add(Dropout(0.5))
model.add(Conv2D(64, (3, 3),
strides = (1,1),
padding = 'valid',
kernel_initializer = 'he_normal',
name = 'conv4'))
model.add(ELU())
model.add(Conv2D(64, (3, 3),
strides= (1,1),
padding = 'valid',
kernel_initializer = 'he_normal',
name = 'conv5'))
model.add(Flatten(name = 'flatten'))
model.add(ELU())
model.add(Dense(100, kernel_initializer = 'he_normal', name = 'fc1'))
model.add(ELU())
model.add(Dense(50, kernel_initializer = 'he_normal', name = 'fc2'))
model.add(ELU())
model.add(Dense(10, kernel_initializer = 'he_normal', name = 'fc3'))
model.add(ELU())
# do not put activation at the end because we want to exact output, not a class identifier
model.add(Dense(1, name = 'output', kernel_initializer = 'he_normal'))
adam = Nadam()
model.compile(optimizer = adam, loss = 'mse')
return model
def build(self,
hidden_layers=[32],
n_filters=[8, 8],
activations=[None],
dropout=0.8,
learning_rate=0.1,
l2_norm=5e-4,
use_bias=False,
k=8):
with self.device:
x = Input(batch_shape=[None, self.n_features],
dtype=tf.float32,
name='features')
adj = Input(batch_shape=[None, None],
dtype=tf.float32,
sparse=False,
name='adj_matrix')
mask = Input(batch_shape=[None], dtype=tf.bool, name='mask')
h = x
for hid, activation in zip(hidden_layers, activations):
h = Dropout(rate=dropout)(h)
h = DenseGraphConv(
hid,
use_bias=use_bias,
activation=activation,
kernel_regularizer=regularizers.l2(l2_norm))([h, adj])
for n_filter in n_filters:
top_k_h = Top_k_features(k=k)([h, adj])
cur_h = LGConvolution(
n_filter,
k,
use_bias=use_bias,
dropout=dropout,
activation=None,
kernel_regularizer=regularizers.l2(l2_norm))(top_k_h)
cur_h = BatchNormalization()(cur_h)
h = Concatenate()([h, cur_h])
h = Dropout(rate=dropout)(h)
h = DenseGraphConv(
self.n_classes,
use_bias=use_bias,
kernel_regularizer=regularizers.l2(l2_norm))([h, adj])
h = tf.boolean_mask(h, mask)
output = Softmax()(h)
model = Model(inputs=[x, adj, mask], outputs=output)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Nadam(lr=learning_rate),
metrics=['accuracy'])
self.k = k
self.model = model
self.built = True
def get_Nadam():
if self.config.Adam_eps == "default":
Adam_eps = 1e-7
elif type(self.config.Adam_eps) == float:
Adam_eps = self.config.Adam_eps
else:
raise Exception('Unsupported Adam_eps!! --> is not default/float')
return Nadam(learning_rate=self.config.learning_rate, epsilon=Adam_eps)
def build(self,
hiddens=[32],
n_filters=[8, 8],
activations=[None, None],
dropout=0.8,
l2_norm=5e-4,
lr=0.1,
use_bias=False,
k=8):
with tf.device(self.device):
x = Input(batch_shape=[None, self.graph.n_attrs],
dtype=self.floatx,
name='attr_matrix')
adj = Input(batch_shape=[None, None],
dtype=self.floatx,
sparse=False,
name='adj_matrix')
mask = Input(batch_shape=[None], dtype=tf.bool, name='node_mask')
h = x
for idx, hidden in enumerate(hiddens):
h = Dropout(rate=dropout)(h)
h = DenseConvolution(
hidden,
use_bias=use_bias,
activation=activations[idx],
kernel_regularizer=regularizers.l2(l2_norm))([h, adj])
for idx, n_filter in enumerate(n_filters):
top_k_h = Top_k_features(k=k)([h, adj])
cur_h = LGConvolution(
n_filter,
kernel_size=k,
use_bias=use_bias,
dropout=dropout,
activation=activations[idx],
kernel_regularizer=regularizers.l2(l2_norm))(top_k_h)
cur_h = BatchNormalization()(cur_h)
h = Concatenate()([h, cur_h])
h = Dropout(rate=dropout)(h)
h = DenseConvolution(
self.graph.n_classes,
use_bias=use_bias,
activation=activations[-1],
kernel_regularizer=regularizers.l2(l2_norm))([h, adj])
h = Mask()([h, mask])
model = Model(inputs=[x, adj, mask], outputs=h)
model.compile(loss=SparseCategoricalCrossentropy(from_logits=True),
optimizer=Nadam(lr=lr),
metrics=['accuracy'])
self.k = k
self.model = model
def main():
# Safety checks
if not os.path.exists(opt.ckptDir):
os.makedirs(opt.ckptDir)
if not os.path.exists(opt.logs):
os.makedirs(opt.logs)
logger.info('Building model...')
model = WDSRConv3D(scale=3,
numFilters=32,
kernelSize=(3, 3, 3),
numResBlocks=8,
expRate=8,
decayRate=0.8,
numImgLR=9,
patchSizeLR=32,
isGrayScale=True)
if opt.optimizer == 'adam':
optimizer = Adam(learning_rate=5e-4)
elif opt.optimizer == 'nadam':
# http://cs229.stanford.edu/proj2015/054_report.pdf
optimizer = Nadam(learning_rate=5e-4)
else:
optimizer = SGD(learning_rate=5e-4)
checkpoint = tf.train.Checkpoint(step=tf.Variable(0),
psnr=tf.Variable(1.0),
optimizer=optimizer,
model=model)
checkpointManager = tf.train.CheckpointManager(checkpoint=checkpoint,
directory=opt.ckptDir,
max_to_keep=5)
# Load Data
logger.info('Loading data...')
patchLR = np.load(opt.patchLR, allow_pickle=True)
patchHR = np.load(opt.patchHR, allow_pickle=True)
X_train, X_val, y_train, y_val = train_test_split(patchLR,
patchHR,
test_size=opt.split,
random_state=17)
valData = [X_val, y_val]
# Initialize metrics
trainLoss = Mean(name='trainLoss')
trainPSNR = Mean(name='trainPSNR')
testLoss = Mean(name='testLoss')
testPSNR = Mean(name='testPSNR')
fitTrainData(model, optimizer, [trainLoss, trainPSNR, testLoss, testPSNR],
shiftCompensatedL1Loss, shiftCompensatedcPSNR, X_train,
y_train, opt.batchSize, opt.epochs, opt.dataBufferSize,
valData, opt.valSteps, checkpoint, checkpointManager,
opt.logDir, opt.ckptDir, opt.saveBestOnly)
def create_model(self):
"""Creates the regressor network.
Returns:
A Keras model for the regressor network architecture.
"""
# Convolutional Encoder
input_img = Input(shape=(*self.dims, self.channels))
conv_1 = Conv2D(32, (3, 3), activation='relu')(input_img)
pool_1 = MaxPooling2D((2, 2))(conv_1)
conv_2 = Conv2D(64, (3, 3), activation='relu')(pool_1)
pool_2 = MaxPooling2D((2, 2), strides=(2, 2))(conv_2)
conv_3 = Conv2D(32, (3, 3), activation='relu')(pool_2)
pool_3 = MaxPooling2D((2, 2))(conv_3)
conv_4 = Conv2D(16, (3, 3), activation='relu')(pool_3)
pool_4 = MaxPooling2D((2, 2))(conv_4)
flatten = Flatten()(pool_4)
# Deep feed-forward network
dense_1_d = Dense(units=300,
activation='relu',
kernel_initializer='he_normal')(flatten)
dropout_1_d = Dropout(self.dropout)(dense_1_d)
dense_2_d = Dense(units=192,
activation='relu',
kernel_initializer='he_normal')(dropout_1_d)
dropout_2_d = Dropout(self.dropout)(dense_2_d)
dense_3_d = Dense(units=123,
activation='relu',
kernel_initializer='he_normal')(dropout_2_d)
dropout_3_d = Dropout(self.dropout)(dense_3_d)
dense_4_d = Dense(units=79,
activation='relu',
kernel_initializer='he_normal')(dropout_3_d)
dropout_4_d = Dropout(self.dropout)(dense_4_d)
dense_5_d = Dense(units=50,
activation='relu',
kernel_initializer='he_normal')(dropout_4_d)
dropout_5_d = Dropout(self.dropout)(dense_5_d)
depth_linear = Dense(units=self.outputs,
activation='linear',
name='depth')(dropout_5_d)
sld_linear = Dense(units=self.outputs, activation='linear',
name='sld')(dropout_5_d)
model = Model(inputs=input_img, outputs=[depth_linear, sld_linear])
model.compile(
loss={
'depth': 'mse',
'sld': 'mse'
}, #msle was tried here for depth.
optimizer=Nadam(self.learning_rate),
metrics={
'depth': 'mae',
'sld': 'mae'
})
return model
def _build_vae(self):
''' builds variational autoencoder network '''
self.kl_anneal = Input(batch_shape=(self.batch_size, ),
name='kl_anneal')
# build VAE
if np.all(np.array([self.alpha, self.beta, self.lamb]) == 0):
self.x_output = self.decoder(self.encoder(self.enc_x_input))
self.vae = Model(inputs=[self.enc_x_input],
outputs=[self.x_output],
name='variational_autoencoder')
elif self.alpha == self.beta == self.lamb:
self.x_output = self.decoder(self.encoder(self.enc_x_input)[2])
self.vae = Model(inputs=[self.enc_x_input, self.kl_anneal],
outputs=[self.x_output],
name='variational_autoencoder')
tc_loss = self.kl_anneal * self.kullback_leibler_divergence_loss()
self.vae.add_loss(tc_loss)
self.vae.add_metric(tc_loss, name='tc_loss', aggregation='mean')
elif np.any(np.array([self.alpha, self.beta, self.lamb]) > 0):
self.x_output = self.decoder(self.encoder(self.enc_x_input)[2])
self.vae = Model(inputs=[self.enc_x_input, self.kl_anneal],
outputs=[self.x_output],
name='variational_autoencoder')
tc_loss = self.kl_anneal * self.total_correlation_loss()
self.vae.add_loss(tc_loss)
self.vae.add_metric(tc_loss, name='tc_loss', aggregation='mean')
# define VAE optimizer
if self.vae_opt_n == 'sgd':
self.vae_opt = SGD(learning_rate=self.lr)
elif self.vae_opt_n == 'sgdm':
self.vae_opt = SGD(learning_rate=self.lr, momentum=0.5)
elif self.vae_opt_n == 'nsgd':
self.vae_opt = SGD(learning_rate=self.lr,
momentum=0.5,
nesterov=True)
elif self.vae_opt_n == 'rmsprop':
self.vae_opt = RMSprop(learning_rate=self.lr)
elif self.vae_opt_n == 'rmsprop_cent':
self.vae_opt = RMSprop(learning_rate=self.lr, centered=True)
elif self.vae_opt_n == 'adam':
self.vae_opt = Adam(learning_rate=self.lr, beta_1=0.5)
elif self.vae_opt_n == 'adam_ams':
self.vae_opt = Adam(learning_rate=self.lr,
beta_1=0.5,
amsgrad=True)
elif self.vae_opt_n == 'adamax':
self.vae_opt = Adamax(learning_rate=self.lr, beta_1=0.5)
elif self.vae_opt_n == 'adamax_ams':
self.vae_opt = Adamax(learning_rate=self.lr,
beta_1=0.5,
amsgrad=True)
elif self.vae_opt_n == 'nadam':
self.vae_opt = Nadam(learning_rate=self.lr, beta_1=0.5)
# compile VAE
rc_loss = self.reconstruction_loss()
self.vae.add_loss(rc_loss)
self.vae.add_metric(rc_loss, name='rc_loss', aggregation='mean')
self.vae.compile(optimizer=self.vae_opt)
def build_network(self):
model = Sequential()
model.add(Dense(24, input_shape=self.observation_space().shape, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(self.action_space().n, activation='linear'))
opt = Nadam(learning_rate=self.alpha)
model.compile(loss='mse', optimizer=opt)
return model
def _init_model(self, hidden_layer_units):
input_ = Input(shape=(self.config.input_shape, ))
y = Dense(units=hidden_layer_units, activation='relu')(input_)
y = Dense(units=self.config.output_shape, activation='sigmoid')(y)
model = Model(input_, y)
model.compile(optimizer=Nadam(),
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
return model
def main():
list_all_midi = get_list_midi()
sampled_200_midi = list_all_midi[0:100] # changed to 100
batch = 1
start_index = 0
note_tokenizer = NoteTokenizer()
seq_len = 50
EPOCHS = 4
BATCH_SONG = 16
BATCH_NNET_SIZE = 96
TOTAL_SONGS = len(sampled_200_midi)
FRAME_PER_SECOND = 5
for i in tqdm(range(len(sampled_200_midi))):
dict_time_notes = generate_dict_time_notes(sampled_200_midi,
batch_song=1,
start_index=i,
use_tqdm=False,
fs=5)
full_notes = process_notes_in_song(dict_time_notes)
for note in full_notes:
note_tokenizer.partial_fit(list(note.values()))
note_tokenizer.add_new_note('e') # Add empty notes
unique_notes = note_tokenizer.unique_word
# print(unique_notes)
model = create_model(seq_len, unique_notes)
print(model.summary())
tf.keras.utils.plot_model(model, 'this_model.png', show_shapes=True)
# TRAIN
optimizer = Nadam()
checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)
checkpoint_dir = '../../training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
loss_fn = sparse_categorical_crossentropy
# TRAIN HERE (1 epoch = 1hour)
train_class = TrainModel(EPOCHS, note_tokenizer, sampled_200_midi,
FRAME_PER_SECOND, BATCH_NNET_SIZE, BATCH_SONG,
optimizer, checkpoint, loss_fn, checkpoint_prefix,
TOTAL_SONGS, model)
train_class.train()
# save mpdel as h5 and the tokenizer
model.save('data/model/epochs4-long-model_ep4.h5')
pickle.dump(note_tokenizer, open("epochs4-long-tokenizer.p", "wb"))
