def construct_VGG19(input_shape=(224, 224, 3), classes=7):
# Stack up the layers
X_Input = Input(input_shape)
# Stage 1
X = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1),
padding='same')(X_Input)
X = BatchNormalization(axis=3)(X)
X = Activation('relu')(X)
X = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1),
padding='same')(X)
X = BatchNormalization(axis=3)(X)
X = Activation('relu')(X)
# Apply Max Pool
X = apply_maxpool(X)
print(X.shape)
# Stage 2
for i in range(2):
X = Conv2D(filters=128,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(X)
X = BatchNormalization(axis=3)(X)
X = Activation('relu')(X)
# Apply Max Pool
X = apply_maxpool(X)
print(X.shape)
# Stage 3
for i in range(4):
X = Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(X)
X = BatchNormalization(axis=3)(X)
X = Activation('relu')(X)
# Apply Max Pool
X = apply_maxpool(X)
print(X.shape)
# Stage 4
for i in range(4):
X = Conv2D(filters=512,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(X)
X = BatchNormalization(axis=3)(X)
X = Activation('relu')(X)
# Apply Max Pool
X = apply_maxpool(X)
print(X.shape)
# Stage 5
for i in range(4):
X = Conv2D(filters=512,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(X)
X = BatchNormalization(axis=3)(X)
X = Activation('relu')(X)
# Apply Max Pool
X = apply_maxpool(X)
print(X.shape)
# Flatten this layer
X = Flatten()(X)
# Dense Layers
for i in range(2):
X = Dense(4096, activation='relu')(X)
# Last layers
X = Dense(classes, activation='softmax')(X)
# Create Model
model = Model(inputs=X_Input, outputs=X)
return model
]
# 输入为*model_body.input, *y_true
# 输出为model_loss
loss_input = [*model_body.output, *y_true]
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': num_classes,
'ignore_thresh': 0.5,
'label_smoothing': label_smoothing
})(loss_input)
model = Model([model_body.input, *y_true], model_loss)
# 训练参数设置
logging = TensorBoard(log_dir=log_dir)
checkpoint = ModelCheckpoint(
log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5',
monitor='val_loss',
save_weights_only=True,
save_best_only=False,
period=1)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=6,
verbose=1)
# 0.1用于验证，0.9用于训练
classes=['NORMAL', 'PNEUMONIA'],
shuffle=False,
batch_size=4)
# define and compile model
base_model = InceptionV3(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
output_layer = Dense(2, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=output_layer)
for layer in base_model.layers:
layer.trainable = False
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
# train and save model
model.fit(train_data_gen,
epochs=10,
validation_data=valid_data_gen,
validation_steps=4,
from tensorflow.keras.models import Sequential, Model from tensorflow.keras.optimizers import Adam from tensorflow.keras.datasets import mnist from tensorflow.keras.activations import relu from keras.utils.np_utils import to_categorical # In[2]: (x_train, y_train), (x_test, y_test) = mnist.load_data() # In[19]: inp_layer = Input(shape=784) x = Dense(64, activation=relu)(inp_layer) out = Dense(10, activation='softmax')(x) model = Model(inputs=[inp_layer], outputs=[out]) # In[20]: model.summary() # In[21]: X_train = x_train.reshape(-1, 28 * 28) X_test = x_test.reshape(-1, 28 * 28) # In[22]: y_train_cat = to_categorical(y_train) y_test_cat = to_categorical(y_test)
def build_q_network(learning_rate: float = 0.00001,
input_shape: tuple = (84, 84),
history_length: int = 4) -> Model:
"""
Builds a dueling DQN as a Keras model. For a good overview of dueling
DQNs (and some motivation behind their use) see:
https://towardsdatascience.com/dueling-deep-q-networks-81ffab672751
Arg:
n_actions: Number of possible action the agent can take
learning_rate: Learning rate
input_shape: Shape of the preprocessed frame the model sees
history_length: Number of historical frames the agent can see
Returns:
A compiled Keras model
"""
# Dueling architecture requires a non-sequential step at the end, so
# Keras's functional API is a natural choice
model_input = Input(shape=(input_shape[0], input_shape[1], history_length))
x = Lambda(lambda layer: layer / 255)(model_input) # normalize by 255
x = Conv2D(32, (8, 8),
strides=4,
kernel_initializer=VarianceScaling(scale=2.),
activation='relu',
use_bias=False)(x)
x = Conv2D(64, (4, 4),
strides=2,
kernel_initializer=VarianceScaling(scale=2.),
activation='relu',
use_bias=False)(x)
x = Conv2D(64, (3, 3),
strides=1,
kernel_initializer=VarianceScaling(scale=2.),
activation='relu',
use_bias=False)(x)
x = Conv2D(1024, (7, 7),
strides=1,
kernel_initializer=VarianceScaling(scale=2.),
activation='relu',
use_bias=False)(x)
# Split into value and advantage streams
val_stream, adv_stream = Lambda(lambda w: tf.split(w, 2, 3))(
x) # custom splitting layer
# State value estimator
val_stream = Flatten()(val_stream)
val = Dense(1, kernel_initializer=VarianceScaling(scale=2.))(val_stream)
# Advantage value estimator
# Each of the four actions has its own advantage value
adv_stream = Flatten()(adv_stream)
adv = Dense(4, kernel_initializer=VarianceScaling(scale=2.))(adv_stream)
# Combine streams into Q-Values
reduce_mean = Lambda(lambda w: tf.reduce_mean(w, axis=1, keepdims=True))
q_vals = Add()([val, Subtract()([adv, reduce_mean(adv)])])
# Build model
model = Model(model_input, q_vals)
model.compile(Adam(learning_rate), loss=tf.keras.losses.Huber())
return model
def make_yolov3_model():
input_image = Input(shape=(None, None, 3))
# Layer 0 => 4
x = _conv_block(input_image, [{
'filter': 32,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 0
}, {
'filter': 64,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 1
}, {
'filter': 32,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 2
}, {
'filter': 64,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 3
}])
# Layer 5 => 8
x = _conv_block(x, [{
'filter': 128,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 5
}, {
'filter': 64,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 6
}, {
'filter': 128,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 7
}])
# Layer 9 => 11
x = _conv_block(x, [{
'filter': 64,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 9
}, {
'filter': 128,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 10
}])
# Layer 12 => 15
x = _conv_block(x, [{
'filter': 256,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 12
}, {
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 13
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 14
}])
# Layer 16 => 36
for i in range(7):
x = _conv_block(x, [{
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 16 + i * 3
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 17 + i * 3
}])
skip_36 = x
# Layer 37 => 40
x = _conv_block(x, [{
'filter': 512,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 37
}, {
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 38
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 39
}])
# Layer 41 => 61
for i in range(7):
x = _conv_block(x, [{
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 41 + i * 3
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 42 + i * 3
}])
skip_61 = x
# Layer 62 => 65
x = _conv_block(x, [{
'filter': 1024,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 62
}, {
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 63
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 64
}])
# Layer 66 => 74
for i in range(3):
x = _conv_block(x, [{
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 66 + i * 3
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 67 + i * 3
}])
# Layer 75 => 79
x = _conv_block(x, [{
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 75
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 76
}, {
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 77
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 78
}, {
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 79
}],
skip=False)
# Layer 80 => 82
yolo_82 = _conv_block(x, [{
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 80
}, {
'filter': 255,
'kernel': 1,
'stride': 1,
'bnorm': False,
'leaky': False,
'layer_idx': 81
}],
skip=False)
# Layer 83 => 86
x = _conv_block(x, [{
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 84
}],
skip=False)
x = UpSampling2D(2)(x)
x = concatenate([x, skip_61])
# Layer 87 => 91
x = _conv_block(x, [{
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 87
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 88
}, {
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 89
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 90
}, {
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 91
}],
skip=False)
# Layer 92 => 94
yolo_94 = _conv_block(x, [{
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 92
}, {
'filter': 255,
'kernel': 1,
'stride': 1,
'bnorm': False,
'leaky': False,
'layer_idx': 93
}],
skip=False)
# Layer 95 => 98
x = _conv_block(x, [{
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 96
}],
skip=False)
x = UpSampling2D(2)(x)
x = concatenate([x, skip_36])
# Layer 99 => 106
yolo_106 = _conv_block(x, [{
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 99
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 100
}, {
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 101
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 102
}, {
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 103
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 104
}, {
'filter': 255,
'kernel': 1,
'stride': 1,
'bnorm': False,
'leaky': False,
'layer_idx': 105
}],
skip=False)
model = Model(input_image, [yolo_82, yolo_94, yolo_106])
return model
from tensorflow.keras.utils import plot_model
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.optimizers import SGD, Adam
from tensorflow.keras.losses import sparse_categorical_crossentropy
import pickle
import numpy as np
import os
from dataset import ptb
import math
# ハイパーパラメータの設定
batch_size = 20
wordvec_size = 100
hidden_size = 100 # RNNの隠れ状態ベクトルの要素数
time_size = 35 # RNNを展開するサイズ
lr = 20.0
max_epoch = 4
max_grad = 0.25
input = Input(batch_shape=(batch_size, None))
output = Embedding(vocab_size, wordvec_size)(input)
output = LSTM(hidden_size,
return_sequences=True,
stateful=True,
)(output)
output = Dense(vocab_size)(output)
model = Model(input, output)
def NeuralNetGeneratorModel(x_train, y_train, x_val, y_val, params):
"""
Keras model for the Neural Network, used to scan the hyperparameter space by Talos
Uses the generator rather than the input data (which are dummies)
"""
# Scaler #
with open(parameters.scaler_path,
'rb') as handle: # Import scaler that was created before
scaler = pickle.load(handle)
# Design network #
# Left branch : classic inputs -> Preprocess -> onehot
inputs_numeric = []
means = []
variances = []
inputs_all = []
encoded_all = []
for idx in range(x_train.shape[1]):
inpName = parameters.inputs[idx].replace('$', '').replace(' ',
'').replace(
'_', '')
input_layer = tf.keras.Input(shape=(1, ), name=inpName)
# Categorical inputs #
if parameters.mask_op[idx]:
operation = getattr(Operations, parameters.operations[idx])()
encoded_all.append(operation(input_layer))
# Numerical inputs #
else:
inputs_numeric.append(input_layer)
means.append(scaler.mean_[idx])
variances.append(scaler.var_[idx])
inputs_all.append(input_layer)
# Concatenate all numerical inputs #
if int(tf_version[1]) < 4:
normalizer = preprocessing.Normalization(name='Normalization')
x_dummy = np.ones((10, len(means)))
# Needs a dummy to call the adapt method before setting the weights
normalizer.adapt(x_dummy)
normalizer.set_weights([np.array(means), np.array(variances)])
else:
normalizer = preprocessing.Normalization(mean=means,
variance=variances,
name='Normalization')
encoded_all.append(
normalizer(tf.keras.layers.concatenate(inputs_numeric,
name='Numerics')))
if len(encoded_all) > 1:
all_features = tf.keras.layers.concatenate(encoded_all,
axis=-1,
name="Features")
else:
all_features = encoded_all[0]
# Right branch : LBN
lbn_input_shape = (len(parameters.LBN_inputs) // 4, 4)
input_lbn_Layer = Input(shape=lbn_input_shape, name='LBN_inputs')
lbn_layer = LBNLayer(
lbn_input_shape,
n_particles=max(params['n_particles'],
1), # Hack so that 0 does not trigger error
boost_mode=LBN.PAIRS,
features=["E", "px", "py", "pz", "pt", "p", "m", "pair_cos"],
name='LBN')(input_lbn_Layer)
batchnorm = tf.keras.layers.BatchNormalization(name='batchnorm')(lbn_layer)
# Concatenation of left and right #
concatenate = tf.keras.layers.Concatenate(axis=-1)(
[all_features, batchnorm])
L1 = Dense(params['first_neuron'],
activation=params['activation'],
kernel_regularizer=l2(params['l2']))(
concatenate if params['n_particles'] > 0 else all_features)
hidden = hidden_layers(params, 1, batch_normalization=True).API(L1)
out = Dense(y_train.shape[1],
activation=params['output_activation'],
name='out')(hidden)
# Tensorboard logs #
# path_board = os.path.join(parameters.main_path,"TensorBoard")
# suffix = 0
# while(os.path.exists(os.path.join(path_board,"Run_"+str(suffix)))):
# suffix += 1
# path_board = os.path.join(path_board,"Run_"+str(suffix))
# os.makedirs(path_board)
# logging.info("TensorBoard log dir is at %s"%path_board)
# Callbacks #
# Early stopping to stop learning if val_loss plateau for too long #
early_stopping = EarlyStopping(**parameters.early_stopping_params)
# Reduce learnign rate in case of plateau #
reduceLR = ReduceLROnPlateau(**parameters.reduceLR_params)
# Custom loss function plot for debugging #
loss_history = LossHistory()
# Tensorboard for checking live the loss curve #
# board = TensorBoard(log_dir=path_board,
# histogram_freq=1,
# batch_size=params['batch_size'],
# write_graph=True,
# write_grads=True,
# write_images=True)
# Callback_list = [loss_history,early_stopping,reduceLR,board]
Callback_list = [loss_history, early_stopping, reduceLR]
# Compile #
if 'resume' not in params: # Normal learning
# Define model #
model_inputs = [inputs_all]
if params['n_particles'] > 0:
model_inputs.append(input_lbn_Layer)
model = Model(inputs=model_inputs, outputs=[out])
initial_epoch = 0
else: # a model has to be imported and resumes training
#custom_objects = {'PreprocessLayer': PreprocessLayer,'OneHot': OneHot.OneHot}
logging.info("Loaded model %s" % params['resume'])
a = Restore(params['resume'],
custom_objects=custom_objects,
method='h5')
model = a.model
initial_epoch = params['initial_epoch']
model.compile(optimizer=Adam(lr=params['lr']),
loss=params['loss_function'],
metrics=[
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.AUC(multi_label=True),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall()
])
model.summary()
# Generator #
training_generator = DataGenerator(
path=parameters.config,
inputs=parameters.inputs,
outputs=parameters.outputs,
inputsLBN=parameters.LBN_inputs if params['n_particles'] > 0 else None,
cut=parameters.cut,
weight=parameters.weight,
batch_size=params['batch_size'],
state_set='training',
model_idx=params['model_idx'] if parameters.crossvalidation else None)
validation_generator = DataGenerator(
path=parameters.config,
inputs=parameters.inputs,
outputs=parameters.outputs,
inputsLBN=parameters.LBN_inputs if params['n_particles'] > 0 else None,
cut=parameters.cut,
weight=parameters.weight,
batch_size=params['batch_size'],
state_set='validation',
model_idx=params['model_idx'] if parameters.crossvalidation else None)
# Some verbose logging #
logging.info("Will use %d workers" % parameters.workers)
logging.warning("Tensorflow location " + tf.__file__)
if len(tf.config.experimental.list_physical_devices('XLA_GPU')) > 0:
logging.info("GPU detected")
#logging.warning(K.tensorflow_backend._get_available_gpus())
# Fit #
history = model.fit_generator(
generator=training_generator, # Training data from generator instance
validation_data=
validation_generator, # Validation data from generator instance
epochs=params['epochs'], # Number of epochs
verbose=1,
max_queue_size=parameters.workers * 2, # Length of batch queue
callbacks=Callback_list, # Callbacks
initial_epoch=
initial_epoch, # In case of resumed training will be different from 0
workers=parameters.
workers, # Number of threads for batch generation (0 : all in same)
shuffle=True, # Shuffle order at each epoch
use_multiprocessing=True) # Needs to be turned on for queuing batches
# Plot history #
PlotHistory(loss_history)
return history, model
def create_model(input_shape, init):
"""
CNN model.
Arguments:
input_shape -- the shape of our input
init -- the weight initialization
Returns:
CNN model
"""
x = inp(shape=input_shape)
x1 = Conv2D(32,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x)
x1 = BatchNormalization()(x1)
x2 = Conv2D(32,
1,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x1)
x2 = BatchNormalization()(x2)
x3 = Concatenate()([x, x2])
x4 = Conv2D(64,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x3)
x4 = BatchNormalization()(x4)
x5 = Conv2D(64,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x4)
x5 = BatchNormalization()(x5)
x6 = Concatenate()([x3, x5])
x7 = Conv2D(96,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x6)
x7 = BatchNormalization()(x7)
x8 = Conv2D(96,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x7)
x8 = BatchNormalization()(x8)
x9 = Concatenate()([x6, x8])
x10 = Conv2D(128,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x9)
x10 = BatchNormalization()(x10)
x11 = Conv2D(128,
3,
activation="relu",
kernel_initializer=init,
bias_regularizer='l2',
padding='same')(x10)
#x8 = Concatenate()([x4,x6])
x11 = BatchNormalization()(x11)
x12 = Concatenate()([x9, x11])
x13 = GlobalAveragePooling2D()(x12)
x14 = Flatten()(x13)
x14 = Reshape((-1, 107))(x14)
x15 = LSTM(1024,
return_sequences=True,
kernel_initializer=initializers.RandomNormal(stddev=0.001),
dropout=0.5,
recurrent_dropout=0.5)(x14)
x16 = LSTM(1024,
go_backwards=True,
return_sequences=False,
kernel_initializer=initializers.RandomNormal(stddev=0.001),
dropout=0.5,
recurrent_dropout=0.5)(x15)
# model = Sequential()
# model.add(Conv2D(32, kernel_size=(5, 5), activation='relu', kernel_initializer = init, bias_regularizer='l2', input_shape=input_shape))
# model.add(BatchNormalization())
# model.add(MaxPooling2D(pool_size=(2,2)))
# model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_initializer = init, bias_regularizer='l2'))
# model.add(BatchNormalization())
# model.add(Conv2D(64, kernel_size=(1, 1), activation='relu', kernel_initializer = init, bias_regularizer='l2'))
# model.add(Conv2D(96, kernel_size=(3, 3), activation='relu', kernel_initializer = init, bias_regularizer='l2'))
# model.add(BatchNormalization())
# model.add(Conv2D(96, kernel_size=(1, 1), activation='relu', kernel_initializer = init, bias_regularizer='l2'))
# model.add(GlobalAveragePooling2D())
# model.add(Flatten())
# model.add(Reshape((-1,8)))
# model.add(Dropout(0.3))
# model.add(LSTM(512, return_sequences=True,
# kernel_initializer=initializers.RandomNormal(stddev=0.001), dropout=0.5, recurrent_dropout=0.5))
# model.add(LSTM(512, go_backwards=True, return_sequences=False,
# kernel_initializer=initializers.RandomNormal(stddev=0.001), dropout=0.5, recurrent_dropout=0.5))
#model.add(LSTM(512, return_sequences=False,
# kernel_initializer=initializers.RandomNormal(stddev=0.001), dropout=0.5, recurrent_dropout=0.5))
#model.add(LSTM(512, go_backwards=True, return_sequences=False,
# kernel_initializer=initializers.RandomNormal(stddev=0.001), dropout=0.5, recurrent_dropout=0.5))
#model.add(BatchNormalization())
x17 = Dropout(0.5)(x16)
x18 = Dense(1, activation='sigmoid', kernel_initializer=init)(x17)
model = Model(inputs=x, outputs=x18)
return model
'''
def createResNetV1(inputShape=(128, 128, 3), numClasses=3):
inputs = Input(shape=inputShape)
v = resLyr(inputs, lyrName='Input')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=16,
numBlocks=3,
downsampleOnFirst=False,
names='Stg1')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=16,
numBlocks=3,
downsampleOnFirst=False,
names='Stg2')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=32,
numBlocks=3,
downsampleOnFirst=True,
names='Stg3')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=64,
numBlocks=3,
downsampleOnFirst=True,
names='Stg4')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=128,
numBlocks=3,
downsampleOnFirst=True,
names='Stg5')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=256,
numBlocks=3,
downsampleOnFirst=True,
names='Stg6')
v = Dropout(0.2)(v)
v = resBlkV1(inputs=v,
numFilters=256,
numBlocks=3,
downsampleOnFirst=False,
names='Stg7')
v = Dropout(0.2)(v)
v = AveragePooling2D(pool_size=8, name='AvgPool')(v)
v = Dropout(0.2)(v)
v = Flatten()(v)
outputs = Dense(numClasses,
activation='softmax',
kernel_initializer=he_normal(33))(v)
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss='categorical_crossentropy',
optimizer=optmz,
metrics=['accuracy'])
return model
def NeuralNetModel(x_train, y_train, x_val, y_val, params):
"""
Keras model for the Neural Network, used to scan the hyperparameter space by Talos
Uses the data provided as inputs
"""
# Split y = [target,weight], Talos does not leave room for the weight so had to be included in one of the arrays
w_train = y_train[:, -1]
w_val = y_val[:, -1]
y_train = y_train[:, :-1]
y_val = y_val[:, :-1]
x_train_lbn = x_train[:, -len(parameters.LBN_inputs):].reshape(
-1, 4,
len(parameters.LBN_inputs) // 4)
x_train = x_train[:, :-len(parameters.LBN_inputs)]
x_val_lbn = x_val[:, -len(parameters.LBN_inputs):].reshape(
-1, 4,
len(parameters.LBN_inputs) // 4)
x_val = x_val[:, :-len(parameters.LBN_inputs)]
# Scaler #
with open(parameters.scaler_path,
'rb') as handle: # Import scaler that was created before
scaler = pickle.load(handle)
# Design network #
# Left branch : classic inputs -> Preprocess -> onehot
inputs_numeric = []
means = []
variances = []
inputs_all = []
encoded_all = []
for idx in range(x_train.shape[1]):
inpName = parameters.inputs[idx].replace('$', '')
input_layer = tf.keras.Input(shape=(1, ), name=inpName)
# Categorical inputs #
if parameters.mask_op[idx]:
operation = getattr(Operations, parameters.operations[idx])()
encoded_all.append(operation(input_layer))
# Numerical inputs #
else:
inputs_numeric.append(input_layer)
means.append(scaler.mean_[idx])
variances.append(scaler.var_[idx])
inputs_all.append(input_layer)
# Concatenate all numerical inputs #
if int(tf_version[1]) < 4:
normalizer = preprocessing.Normalization(name='Normalization')
x_dummy = np.ones((10, len(means)))
# Needs a dummy to call the adapt method before setting the weights
normalizer.adapt(x_dummy)
normalizer.set_weights([np.array(means), np.array(variances)])
else:
normalizer = preprocessing.Normalization(mean=means,
variance=variances,
name='Normalization')
encoded_all.append(
normalizer(tf.keras.layers.concatenate(inputs_numeric,
name='Numerics')))
if len(encoded_all) > 1:
all_features = tf.keras.layers.concatenate(encoded_all,
axis=-1,
name="Features")
else:
all_features = encoded_all[0]
# Right branch : LBN
input_lbn_Layer = Input(shape=x_train_lbn.shape[1:], name='LBN_inputs')
lbn_layer = LBNLayer(
x_train_lbn.shape[1:],
n_particles=max(params['n_particles'],
1), # Hack so that 0 does not trigger error
boost_mode=LBN.PAIRS,
features=["E", "px", "py", "pz", "pt", "p", "m", "pair_cos"],
name='LBN')(input_lbn_Layer)
batchnorm = tf.keras.layers.BatchNormalization(name='batchnorm')(lbn_layer)
# Concatenation of left and right #
concatenate = tf.keras.layers.Concatenate(axis=-1)(
[all_features, batchnorm])
L1 = Dense(params['first_neuron'],
activation=params['activation'],
kernel_regularizer=l2(params['l2']))(
concatenate if params['n_particles'] > 0 else all_features)
hidden = hidden_layers(params, 1, batch_normalization=True).API(L1)
out = Dense(y_train.shape[1],
activation=params['output_activation'],
name='out')(hidden)
# Check preprocessing #
preprocess = Model(inputs=inputs_numeric, outputs=encoded_all[-1])
x_numeric = x_train[:, [not m for m in parameters.mask_op]]
out_preprocess = preprocess.predict(np.hsplit(x_numeric,
x_numeric.shape[1]),
batch_size=params['batch_size'])
mean_scale = np.mean(out_preprocess)
std_scale = np.std(out_preprocess)
if abs(mean_scale) > 0.01 or abs(
(std_scale - 1) /
std_scale) > 0.1: # Check that scaling is correct to 1%
logging.warning(
"Something is wrong with the preprocessing layer (mean = %0.6f, std = %0.6f), maybe you loaded an incorrect scaler"
% (mean_scale, std_scale))
# Tensorboard logs #
#path_board = os.path.join(parameters.main_path,"TensorBoard")
#suffix = 0
#while(os.path.exists(os.path.join(path_board,"Run_"+str(suffix)))):
# suffix += 1
#path_board = os.path.join(path_board,"Run_"+str(suffix))
#os.makedirs(path_board)
#logging.info("TensorBoard log dir is at %s"%path_board)
# Callbacks #
# Early stopping to stop learning if val_loss plateau for too long #
early_stopping = EarlyStopping(**parameters.early_stopping_params)
# Reduce learnign rate in case of plateau #
reduceLR = ReduceLROnPlateau(**parameters.reduceLR_params)
# Custom loss function plot for debugging #
loss_history = LossHistory()
# Tensorboard for checking live the loss curve #
#board = TensorBoard(log_dir=path_board,
# histogram_freq=1,
# batch_size=params['batch_size'],
# write_graph=True,
# write_grads=True,
# write_images=True)
Callback_list = [loss_history, early_stopping, reduceLR]
# Compile #
if 'resume' not in params: # Normal learning
# Define model #
model_inputs = [inputs_all]
if params['n_particles'] > 0:
model_inputs.append(input_lbn_Layer)
model = Model(inputs=model_inputs, outputs=[out])
initial_epoch = 0
else: # a model has to be imported and resumes training
#custom_objects = {'PreprocessLayer': PreprocessLayer,'OneHot': OneHot.OneHot}
logging.info("Loaded model %s" % params['resume'])
a = Restore(params['resume'],
custom_objects=custom_objects,
method='h5')
model = a.model
initial_epoch = params['initial_epoch']
model.compile(optimizer=Adam(lr=params['lr']),
loss=params['loss_function'],
metrics=[
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.AUC(multi_label=True),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall()
])
model.summary()
fit_inputs = np.hsplit(x_train, x_train.shape[1])
fit_val = (np.hsplit(x_val, x_val.shape[1]), y_val, w_val)
if params['n_particles'] > 0:
fit_inputs.append(x_train_lbn)
fit_val[0].append(x_val_lbn)
# Fit #
history = model.fit(x=fit_inputs,
y=y_train,
sample_weight=w_train,
epochs=params['epochs'],
batch_size=params['batch_size'],
verbose=1,
validation_data=fit_val,
callbacks=Callback_list)
# Plot history #
PlotHistory(loss_history, params)
return history, model
def main():
# Hyperparameters
batch = 4
learning_rate = 0.001
patience = 5
weights_path = './weights'
epochs = 50
load_pretrained = None
input_size = (224, 224, 3)
# Load dataset
train_generator, valid_generator = HockeyFightDataset(
batch=batch, size=input_size).dataset()
# Modeling
inputs = Input([None, *input_size])
predictions, end_points = inceptionI3D(inputs,
dropout_keep_prob=0.5,
final_endpoint='Predictions')
i3d_model = Model(inputs, predictions)
i3d_model.compile(optimizer=Adam(lr=learning_rate),
loss='categorical_crossentropy',
metrics=['acc'])
# Callbacks
callbacks = []
tensorboard = TensorBoard()
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=patience,
verbose=1,
mode='min',
min_lr=1e-6)
model_checkpoint = ModelCheckpoint(os.path.join(
weights_path, f'I3D_{batch}batch_{epochs}epochs.h5'),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='min')
callbacks.append(tensorboard)
callbacks.append(reduce_lr)
callbacks.append(model_checkpoint)
# Train!
i3d_model.fit_generator(generator=train_generator,
steps_per_epoch=get_steps_hockey(900, batch),
epochs=epochs,
callbacks=callbacks,
validation_data=valid_generator,
validation_steps=get_steps_hockey(100, batch),
use_multiprocessing=True,
workers=-1)
# Evaluate
evaluation = i3d_model.evaluate_generator(generator=valid_generator)
print(
f'Evaluation loss : {evaluation["loss"]} , acc : {evaluation["acc"]}')
def res_unet(input_shape=(1024, 1024, 3), chan_num=3):
inputs = Input(shape=input_shape)
# 1024
down0b = conv_layer(inputs, 16, 2, strides=2)
down0b = conv_layer(down0b, 16, 2)
down0b_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0b)
# down0b_pool = conv_layer(inputs, 32, 4, strides = 4, shape= 5)
# 256
down0a = conv_layer(down0b_pool, 32, 4)
down0a = conv_layer(down0a, 32, 4)
down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)
down0a_pool = add_res_dwn(down0a_pool, down0b_pool, 32) # res coonect
# 128
down0 = conv_layer(down0a_pool, 64, 4)
down0 = conv_layer(down0, 64, 4)
down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
down0_pool = add_res_dwn(down0_pool, down0a_pool, 64) # res coonect
# 64
down1 = conv_layer(down0_pool, 128, 8)
down1 = conv_layer(down1, 128, 8)
down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
down1_pool = add_res_dwn(down1_pool, down0_pool, 128) # res coonect
# 32
down2 = conv_layer(down1_pool, 256, 8)
down2 = conv_layer(down2, 256, 8)
down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
down2_pool = add_res_dwn(down2_pool, down1_pool, 256) # res coonect
# 16
down3 = conv_layer(down2_pool, 512, 16)
down3 = conv_layer(down3, 512, 16)
down3 = add_res(down3, down2_pool, 512)
up2 = UpSampling2D((2, 2))(down3)
up2_c = concatenate([down2, up2], axis=chan_num)
up2 = conv_layer(up2_c, 256, 8)
up2 = conv_layer(up2, 256, 8)
up2 = add_res(up2, up2_c, 256)
# 32
up1 = UpSampling2D((2, 2))(up2)
up1_c = concatenate([down1, up1], axis=chan_num)
up1 = conv_layer(up1_c, 128, 8)
up1 = conv_layer(up1, 128, 8)
up1 = add_res(up1, up1_c, 128)
# 64
up0 = UpSampling2D((2, 2))(up1)
up0_c = concatenate([down0, up0], axis=chan_num)
up0 = conv_layer(up0_c, 64, 4)
up0 = conv_layer(up0, 64, 4)
up0 = add_res(up0, up0_c, 64)
# 128
up0a = UpSampling2D((2, 2))(up0)
up0a_c = concatenate([down0a, up0a], axis=chan_num)
up0a = conv_layer(up0a_c, 32, 4)
up0a = conv_layer(up0a, 32, 4)
up0a = add_res(up0a, up0a_c, 32)
# 256
# up0b = UpSampling2D((2, 2))(up0a)
# up0b_c = concatenate([down0b, up0b], axis=chan_num)
up0b_c = UpSampling2D((2, 2))(up0a)
up0b = conv_layer(up0b_c, 8, 2)
up0b = add_res(up0b, up0b_c, 8)
# 512
# Crypt predict
crypt_fufi = Conv2D(1, (1, 1))(up0b)
crypt_fufi = layers.Activation('sigmoid', dtype='float32',
name='crypt')(crypt_fufi)
# just unet
just_unet = Model(inputs=inputs, outputs=[crypt_fufi, up0a])
return just_unet
def construct_Resnet18(input_shape=(32, 32, 3), classes=10):
"""
Implementation of the popular Resnet18 the following architecture:
CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3
-> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER
Arguments:
input_shape -- shape of the images of the dataset
classes -- integer, number of classes
Returns:
model -- a Model() instance in tensorflow.keras
"""
# Define input as a tensor of input shape
X_input = Input(input_shape)
X = X_input
# Zero Padding
#X = ZeroPadding2D((3, 3))(X_input)
# Stage 1
X = Conv2D(64, (7, 7),
strides=(2, 2),
name='conv1',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name='bn_conv1')(X)
X = Activation('relu')(X)
#X = MaxPooling2D((3, 3), strides=(2, 2))(X)
# Stage 2
# X = convolutional_block(
# X_input, f=3, filters=[64, 64], stage=2, block='a', s=1)
X = identity_block(X, 3, [64, 64], stage=2, block='b')
X = identity_block(X, 3, [64, 64], stage=2, block='c')
X = identity_block(X, 3, [64, 64], stage=2, block='d')
# Stage 3
X = convolutional_block(X, f=3, filters=[64, 128], stage=3, block='a', s=2)
X = identity_block(X, 3, [64, 128], stage=3, block='b')
X = identity_block(X, 3, [64, 128], stage=3, block='c')
# Stage 4
X = convolutional_block(X,
f=3,
filters=[128, 256],
stage=4,
block='a',
s=2)
X = identity_block(X, 3, [128, 256], stage=4, block='b')
X = identity_block(X, 3, [128, 256], stage=4, block='c')
# Stage 5
# X = convolutional_block(
# X, f=3, filters=[256, 512], stage=5, block='a', s=2)
# X = identity_block(X, 3, [256, 512], stage=5, block='b')
# X = identity_block(X, 3, [256, 512], stage=5, block='c')
# AVGPOOL
X = AveragePooling2D((1, 1))(X)
# Output Layer
X = Flatten()(X)
# X = Dense(1000, activation='relu', name='fc10000',
# kernel_initializer=glorot_uniform(seed=0))(X)
X = Dense(classes,
activation='softmax',
name='fc' + str(classes),
kernel_initializer=glorot_uniform(seed=0))(X)
# Create model
model = Model(inputs=X_input, outputs=X, name='Resnet18')
return model
def triplet_loss(y_true, y_pred, alpha = ALPHA):
anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]
pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), axis=-1)
neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), axis=-1)
basic_loss = tf.add(tf.subtract(pos_dist, neg_dist), alpha)
loss = tf.reduce_sum(tf.maximum(basic_loss, 0.0))
return loss
PATH_TO_DLIB_H5_MODEL = 'model/dlib_face_recognition_resnet_model_v1.h5'
# load model
model_from_file = tf.keras.models.load_model(PATH_TO_DLIB_H5_MODEL, custom_objects={'ScaleLayer': ScaleLayer, 'ReshapeLayer': ReshapeLayer})
top_model = Sequential()
top_model.add(Dense(128, input_shape=(128,), use_bias = False))
model = Model(inputs=model_from_file.input, outputs=top_model(model_from_file.output))
model.summary()
for layer in model.layers[:-1]:
layer.trainable = False
# for layer in model.layers:
# layer.trainable = True
IMAGE_SIZE = 150
NUM_EPOCHS = 30
STEPS_PER_EPOCH = 10
input_shape=( IMAGE_SIZE, IMAGE_SIZE, 3)
A = Input(shape=input_shape, name = 'anchor')
P = Input(shape=input_shape, name = 'anchorPositive')
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input
from tensorflow.keras.utils import plot_model
from model_blocks import optimized_inception_module
input = Input(shape=(256, 256, 3))
# Add inception block
layer = optimized_inception_module(input, 64, 96, 128, 16, 32, 32)
# Add inception block
layer = optimized_inception_module(layer, 128, 128, 192, 32, 96, 64)
model = Model(inputs=input, outputs=layer)
model.summary()
plot_model(model,
show_shapes=True,
to_file="artifacts/multi_inception_model.png")
correct = K.cast( K.equal(targ,pred), dtype='float32') #cast bool tensor to float
# 0 is padding, don't include those- mask is tensor representing non-pad value
mask = K.cast(K.greater(targ, 0), dtype='float32') #cast bool-tensor to float
n_correct = K.sum(mask * correct) #
n_total = K.sum(mask)
return n_correct / n_total
#custom loss because we dont want to consider padding
def loss(y_true, y_pred):
# both are of shape ( _, Ty, DEOCDER_VOCAB_SIZE )
mask = K.cast(y_true > 0, dtype='float32')
out = mask * y_true * K.log(y_pred) #cross entopy loss
return -K.sum(out) / K.sum(mask)
model = Model( [ encoder_inp, decoder_inp, initial_decoder_h, initial_decoder_c ], outputs )
model.compile( optimizer="rmsprop", loss=loss, metrics=[acc])
print(model.summary())
#to convert a batch of decoder outputs to one hot.
def ohe_decoder_output( x ):
ohe = np.zeros( (*(x.shape), DECODER_VOCAB_SIZE) )
for i in range(len(x)):
for j in range(MAX_DECODER_SIZE):
ohe[i,j,x[i][j]] = 1
return ohe
#Using generator due to huge volume of data.
def generator( encoder_inp, decoder_inp, decoder_out, bs=128 ):
total_batches = len(encoder_inp)//bs
def QNN_model(n_classes):
kern = 8
n_layers = 9
inputs = Input(shape=(98, 40))
# First QConv1D layer
x = QuaternionConv1D(kern,
2,
strides=1,
activation="relu",
padding="valid",
use_bias=True)(inputs)
x = PReLU()(x)
# Second QConv1D layer
x = QuaternionConv1D(kern * 2,
2,
strides=1,
activation="relu",
padding="valid",
use_bias=True)(x)
x = PReLU()(x)
# Third QConv1D layer
x = QuaternionConv1D(kern * 4,
2,
strides=1,
activation="relu",
padding="valid",
use_bias=True)(x)
x = PReLU()(x)
# Fourth QConv1D layer
x = QuaternionConv1D(kern * 8,
2,
strides=1,
activation="relu",
padding="valid",
use_bias=True)(x)
x = PReLU()(x)
"""
# Conv 1D layers (1-3)
for i in range(n_layers//3):
x = QuaternionConv1D(kern*2, 2, strides=1, activation="relu", padding="valid", use_bias=True)(x)
x = PReLU()(x)
# Conv 1D layers (4-6)
for i in range(n_layers//3):
x = QuaternionConv1D(kern*4, 2, strides=1, activation="relu", padding="valid", use_bias=True)(x)
x = PReLU()(x)
# Conv 1D layers (7-9)
for i in range(n_layers//3):
x = QuaternionConv1D(kern*8, 2, strides=1, activation="relu", padding="valid", use_bias=True)(x)
x = PReLU()(x)
"""
# FLatten layer
flat = Flatten()(x)
# Dense layer 1
dense = QuaternionDense(256, activation='relu')(flat)
# Dense layer 2
dense2 = QuaternionDense(256, activation='relu')(dense)
# Dense layer 2
dense3 = QuaternionDense(256, activation='relu')(dense2)
outputs = Dense(n_classes, activation='softmax')(dense3)
model = Model(inputs=inputs, outputs=outputs)
model.summary()
return model
return model
# In[9]: from tensorflow.keras.models import Model from tensorflow.keras.layers import Dense, concatenate from tensorflow.keras.layers import Input # define two sets of inputs low_rf = Input(shape=(X.shape[2],)) high_rf = Input(shape=(X.shape[2],)) # the first branch operates on the first input x1 = Dense(500 , activation="relu")(low_rf) # x1 = Dense(500, activation="relu")(x1) x1 = Model(inputs=low_rf, outputs=x1) # the second branch operates on the second input x2 = Dense(500 , activation="relu")(high_rf) # x2 = Dense(500, activation="relu")(x2) x2 = Model(inputs=high_rf, outputs=x2) # combine the output of the two branches combined = concatenate([x1.output, x2.output]) # apply a FC layer and then a regression prediction on the # combined outputs z = Dense(100, activation="relu")(combined) z = Dense(1, activation="sigmoid")(z) model = Model(inputs=[x1.input, x2.input], outputs=z)
def nba_ARMA(node2vec_dim):
channels = 30
node2vec_input = Input(shape=(62, node2vec_dim))
node2vec_Veg_input = Input(shape=(31, node2vec_dim))
A_input = Input(shape=(62, 62))
A_Veg_input = Input(shape=(31, 31))
team_inputs = Input(shape=(2, ), dtype=tf.int64)
line_input = Input(shape=(1, ))
last_5_input = Input(shape=(10, ))
one_hot_input = Input(shape=(60, ))
ARMA = spektral.layers.ARMAConv(
channels,
order=4,
iterations=1,
share_weights=False,
gcn_activation='relu',
dropout_rate=0.2,
activation='elu',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([node2vec_input, A_input])
ARMA_Veg = spektral.layers.ARMAConv(
channels,
order=4,
iterations=1,
share_weights=False,
gcn_activation='relu',
dropout_rate=0.2,
activation='elu',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([node2vec_Veg_input, A_Veg_input])
#extracts nodes for link prediction
game_vec = extract_team_GAT.Game_Vec(channels)(
[team_inputs, ARMA, ARMA_Veg])
rshp = Reshape((int(np.floor(6 * channels)), ))(game_vec)
cat = Concatenate()([rshp, one_hot_input])
dense1 = Dense(int(np.floor(6.5 * channels)), activation='tanh')(cat)
drop1 = Dropout(.05)(dense1)
dense2 = Dense(int(np.floor(2 * channels)), activation='tanh')(drop1)
drop2 = Dropout(.05)(dense2)
drop2 = Concatenate()([drop2, last_5_input])
dense3 = Dense(int(np.floor(channels / 2)))(drop2)
drop3 = Dropout(.05)(dense3)
add_line = Concatenate()([drop3, line_input])
prediction = Dense(1)(add_line)
model = Model(inputs=[
team_inputs, line_input, node2vec_input, A_input, node2vec_Veg_input,
A_Veg_input, last_5_input, one_hot_input
],
outputs=prediction)
return model
import tensorflow as tf
os.environ["CUDA_VISIBLE_DEVICES"] = "6"
#gpu_options = tf.GPUOptions(allow_growth=True)
#sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
from keras.utils import plot_model
from matplotlib import pyplot as plt
#【0】MobileNet模型，加载预训练权重
base_model = MobileNet(weights='imagenet')
#print(base_model.summary())
#【1】创建一个新model, 使得它的输出(outputs)是 MobileNet 中任意层的输出(output)
#定义层的时候可以指定name参数，如：x=Dense(784,activation='relu',name="my_lay")(x)
model = Model(inputs=base_model.input,
outputs=base_model.get_layer('dropout').output)
print(model.summary()) # 打印模型概况
#【2】从网上下载一张图片，保存在当前路径下
img_path = 'elephant.jpg'
img = image.load_img(img_path, target_size=(224, 224)) # 加载图片并resize成224x224
#【3】将图片转化为4d tensor形式
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
#【4】数据预处理
x = preprocess_input(x) #去均值中心化，preprocess_input函数详细功能见注释
#【5】提取特征
block4_pool_features = model.predict(x)
def discriminator(node2vec_dim):
channels = 40
feature_input = Input(shape=(62, node2vec_dim))
feature_Veg_input = Input(shape=(31, node2vec_dim))
feature_M_input = Input(shape=(31, node2vec_dim))
A_input = Input(shape=(62, 62))
A_Veg_input = Input(shape=(31, 31))
M_Graph_input = Input(shape=(31, 31))
team_inputs = Input(shape=(2, ), dtype=tf.int64)
line_input = Input(shape=(1, ))
model_input = Input(shape=(1, ))
last_5_input = Input(shape=(10, ))
one_hot_input = Input(shape=(60, ))
A_input_sp = extract_team_GAT.To_Sparse()(A_input)
A_Veg_input_sp = extract_team_GAT.To_Sparse()(A_Veg_input)
M_Graph_input_sp = extract_team_GAT.To_Sparse()(M_Graph_input)
GIN = spektral.layers.GINConv(
channels,
epsilon=None,
mlp_hidden=[channels, channels],
mlp_activation='relu',
aggregate='sum',
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([feature_input, A_input_sp])
GIN_Veg = spektral.layers.GINConv(
channels,
epsilon=None,
mlp_hidden=[channels, channels],
mlp_activation='relu',
aggregate='sum',
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([feature_Veg_input, A_Veg_input_sp])
GIN_M = spektral.layers.GINConv(
channels,
epsilon=None,
mlp_hidden=[channels, channels],
mlp_activation='relu',
aggregate='sum',
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([feature_M_input, M_Graph_input_sp])
game_vec = extract_team_GAT.Game_Vec_D(channels)(
[team_inputs, GIN, GIN_Veg, GIN_M])
rshp = Reshape((int(np.floor(8 * channels)), ))(game_vec)
cat = Concatenate()([rshp, one_hot_input, line_input, model_input])
dense1 = Dense(int(np.floor(8.5 * channels)), activation='tanh')(cat)
drop1 = Dropout(.01)(dense1)
dense2 = Dense(int(np.floor(4 * channels)), activation='tanh')(drop1)
drop2 = Dropout(.01)(dense2)
drop2 = Concatenate()([drop2, last_5_input])
dense3 = Dense(int(np.floor(channels / 2)))(drop2)
drop3 = Dropout(.01)(dense3)
prediction = Dense(2, activation='softmax')(drop3)
#extracts nodes for link prediction
model = Model(inputs=[
team_inputs, line_input, model_input, feature_input, A_input,
feature_Veg_input, A_Veg_input, feature_M_input, M_Graph_input,
last_5_input, one_hot_input
],
outputs=prediction)
return model
flat = Flatten()(pool2)
dense = Dense(128, activation = 'relu')(flat)
output = Dense(10, activation = 'softmax', name = "output_node")(dense)
return output
#define input layer
inpt = Input(shape = (28,28,1), name = "input_node")
#call the model
logits = CNN(inpt)
#define model
model = Model(inpt,logits)
#compile the model
model.compile(optimizer = keras.optimizers.Adam(lr = 0.0001), \
loss = 'categorical_crossentropy', metrics = ['accuracy'])
#convert to an Estimator
# the model_dir states where the graph and checkpoint files will be saved to
estimator_model = tf.keras.estimator.model_to_estimator(keras_model = model, \
model_dir = './models')
"""
input_function: features random shuffle batch and epoches
"""
def input_function(features,labels=None,shuffle=False):
def nba_gen(node2vec_dim):
channels = 40
node2vec_input = Input(shape=(62, node2vec_dim))
node2vec_Veg_input = Input(shape=(31, node2vec_dim))
A_input = Input(shape=(62, 62))
A_Veg_input = Input(shape=(31, 31))
A_input_sp = extract_team_GAT.To_Sparse()(A_input)
A_Veg_input_sp = extract_team_GAT.To_Sparse()(A_Veg_input)
team_inputs = Input(shape=(2, ), dtype=tf.int64)
line_input = Input(shape=(1, ))
last_5_input = Input(shape=(10, ))
one_hot_input = Input(shape=(60, ))
conv = spektral.layers.GeneralConv(
channels=channels,
batch_norm=True,
dropout=0.0,
aggregate='sum',
activation='relu',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([node2vec_input, A_input_sp])
conv_veg = spektral.layers.GeneralConv(
channels=channels,
batch_norm=True,
dropout=0.0,
aggregate='sum',
activation='relu',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)([node2vec_Veg_input, A_Veg_input_sp])
#extracts nodes for link prediction
game_vec = extract_team_GAT.Game_Vec(channels)(
[team_inputs, conv, conv_veg])
rshp = Reshape((int(np.floor(6 * channels)), ))(game_vec)
cat = Concatenate()([rshp, one_hot_input])
dense1 = Dense(int(np.floor(6.5 * channels)), activation='tanh')(cat)
drop1 = Dropout(.01)(dense1)
dense2 = Dense(int(np.floor(2 * channels)), activation='tanh')(drop1)
drop2 = Dropout(.01)(dense2)
drop2 = Concatenate()([drop2, last_5_input])
dense3 = Dense(int(np.floor(channels / 2)))(drop2)
drop3 = Dropout(.01)(dense3)
add_line = Concatenate()([drop3, line_input])
prediction = Dense(1)(add_line)
model = Model(inputs=[
team_inputs, line_input, node2vec_input, A_input, node2vec_Veg_input,
A_Veg_input, last_5_input, one_hot_input
],
outputs=prediction)
return model
# Get the InceptionV3 model so we can do transfer learning
base_inception = InceptionV3(weights='imagenet',
include_top=False,
input_shape=input_shape)
# Make all layers untrainable
for layer in base_inception.layers:
layer.trainable = False
#last_output = base_inception.output
last_output = base_inception.get_layer('mixed7').output
x = Flatten()(last_output)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(2, activation='softmax')(x)
model = Model(inputs=base_inception.input, outputs=predictions)
inception_layers = [(layer, layer.name, layer.trainable)
for layer in model.layers]
print(
pd.DataFrame(inception_layers,
columns=['Layer Type', 'Layer Name', 'Layer Trainable']))
model.compile(Adam(),
loss='categorical_crossentropy',
metrics=['accuracy'])
#model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
history = model.fit(train_datagen,
steps_per_epoch=len(train_datagen.labels) //
batch_size,
def build_and_load_model(model_capacity):
"""
Build the CNN model and load the weights
Parameters
----------
model_capacity : 'tiny', 'small', 'medium', 'large', or 'full'
String specifying the model capacity, which determines the model's
capacity multiplier to 4 (tiny), 8 (small), 16 (medium), 24 (large),
or 32 (full). 'full' uses the model size specified in the paper,
and the others use a reduced number of filters in each convolutional
layer, resulting in a smaller model that is faster to evaluate at the
cost of slightly reduced pitch estimation accuracy.
Returns
-------
model : tensorflow.keras.models.Model
The pre-trained keras model loaded in memory
"""
from tensorflow.keras.layers import Input, Reshape, Conv2D, BatchNormalization
from tensorflow.keras.layers import MaxPool2D, Dropout, Permute, Flatten, Dense
from tensorflow.keras.models import Model
if models[model_capacity] is None:
capacity_multiplier = {
'tiny': 4,
'small': 8,
'medium': 16,
'large': 24,
'full': 32
}[model_capacity]
layers = [1, 2, 3, 4, 5, 6]
filters = [n * capacity_multiplier for n in [32, 4, 4, 4, 8, 16]]
widths = [512, 64, 64, 64, 64, 64]
strides = [(4, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)]
x = Input(shape=(1024, ), name='input', dtype='float32')
y = Reshape(target_shape=(1024, 1, 1), name='input-reshape')(x)
for l, f, w, s in zip(layers, filters, widths, strides):
y = Conv2D(f, (w, 1),
strides=s,
padding='same',
activation='relu',
name="conv%d" % l)(y)
y = BatchNormalization(name="conv%d-BN" % l)(y)
y = MaxPool2D(pool_size=(2, 1),
strides=None,
padding='valid',
name="conv%d-maxpool" % l)(y)
y = Dropout(0.25, name="conv%d-dropout" % l)(y)
y = Permute((2, 1, 3), name="transpose")(y)
y = Flatten(name="flatten")(y)
y = Dense(360, activation='sigmoid', name="classifier")(y)
model = Model(inputs=x, outputs=y)
package_dir = os.path.dirname(os.path.realpath(__file__))
filename = "model-{}.h5".format(model_capacity)
model.load_weights(os.path.join(package_dir, filename))
model.compile('adam', 'binary_crossentropy')
models[model_capacity] = model
return models[model_capacity]
# K = number of output units # Make some data N = 1 T = 10 D = 3 K = 2 X = np.random.randn(N, T, D) # Make an RNN M = 5 # number of hidden units i = Input(shape=(T, D)) x = SimpleRNN(M)(i) x = Dense(K)(x) model = Model(i, x) # Get the output Yhat = model.predict(X) print(Yhat) # See if we can replicate this output # Get the weights first model.summary() # See what's returned model.layers[1].get_weights() # Check their shapes # Should make sense # First output is input > hidden
x_red_24 = conv2d(x_red_24, 256, 3, 2, 'valid', True, name='x_red2_c33')
x = Concatenate(axis=3,
name='red_concat_2')([x_red_21, x_red_22, x_red_23, x_red_24])
#Inception-ResNet-C modules
x = incresC(x, 0.2, name='incresC_1')
x = incresC(x, 0.2, name='incresC_2')
x = incresC(x, 0.2, name='incresC_3')
#TOP
x = GlobalAveragePooling2D(data_format='channels_last')(x)
x = Dropout(0.6)(x)
x = Dense(num_classes, activation='softmax')(x)
model = Model(img_input, x, name='inception_resnet_v2')
model.compile(Adam(lr=0.0001),
loss='binary_crossentropy',
metrics=['accuracy'])
#model.summary()
#for TPU
'''
tpu_model = tf.contrib.tpu.keras_to_tpu_model(
model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
)
)
def create_hippmapp3r_unet_model_3d(input_image_size,
do_first_network=True,
data_format="channels_last"):
"""
Implementation of the "HippMapp3r" U-net architecture
Creates a keras model implementation of the u-net architecture
described here:
https://onlinelibrary.wiley.com/doi/pdf/10.1002/hbm.24811
with the implementation available here:
https://github.com/mgoubran/HippMapp3r
Arguments
---------
input_image_size : tuple of length 4
Used for specifying the input tensor shape. The shape (or dimension) of
that tensor is the image dimensions followed by the number of channels
(e.g., red, green, and blue).
do_first_network : boolean
Boolean dictating if the model built should be the first (initial) network
or second (refinement) network.
data_format : string
One of "channels_first" or "channels_last". We do this for this specific
architecture as the original weights were saved in "channels_first" format.
Returns
-------
Keras model
A 3-D keras model defining the U-net network.
Example
-------
>>> shape_initial_stage = (160, 160, 128)
>>> model_initial_stage = antspynet.create_hippmapp3r_unet_model_3d((*shape_initial_stage, 1), True)
>>> model_initial_stage.load_weights(antspynet.get_pretrained_network("hippMapp3rInitial"))
>>> shape_refine_stage = (112, 112, 64)
>>> model_refine_stage = antspynet.create_hippmapp3r_unet_model_3d((*shape_refine_stage, 1), False)
>>> model_refine_stage.load_weights(antspynet.get_pretrained_network("hippMapp3rRefine"))
"""
channels_axis = None
if data_format == "channels_last":
channels_axis = 4
elif data_format == "channels_first":
channels_axis = 1
else:
raise ValueError("Unexpected string for data_format.")
def convB_3d_layer(input, number_of_filters, kernel_size=3, strides=1):
block = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
data_format=data_format)(input)
block = InstanceNormalization(axis=channels_axis)(block)
block = LeakyReLU()(block)
return (block)
def residual_block_3d(input, number_of_filters):
block = convB_3d_layer(input, number_of_filters)
block = SpatialDropout3D(rate=0.3, data_format=data_format)(block)
block = convB_3d_layer(block, number_of_filters)
return (block)
def upsample_block_3d(input, number_of_filters):
block = UpSampling3D(data_format=data_format)(input)
block = convB_3d_layer(block, number_of_filters)
return (block)
def feature_block_3d(input, number_of_filters):
block = convB_3d_layer(input, number_of_filters)
block = convB_3d_layer(block, number_of_filters, kernel_size=1)
return (block)
number_of_filters_at_base_layer = 16
number_of_layers = 6
if do_first_network == False:
number_of_layers = 5
inputs = Input(shape=input_image_size)
# Encoding path
add = None
encoding_convolution_layers = []
for i in range(number_of_layers):
number_of_filters = number_of_filters_at_base_layer * 2**i
conv = None
if i == 0:
conv = convB_3d_layer(inputs, number_of_filters)
else:
conv = convB_3d_layer(add, number_of_filters, strides=2)
residual_block = residual_block_3d(conv, number_of_filters)
add = Add()([conv, residual_block])
encoding_convolution_layers.append(add)
# Decoding path
outputs = encoding_convolution_layers[number_of_layers - 1]
# 256
number_of_filters = (number_of_filters_at_base_layer *
2**(number_of_layers - 2))
outputs = upsample_block_3d(outputs, number_of_filters)
if do_first_network == True:
# 256, 128
outputs = Concatenate(axis=channels_axis)(
[encoding_convolution_layers[4], outputs])
outputs = feature_block_3d(outputs, number_of_filters)
number_of_filters = int(number_of_filters / 2)
outputs = upsample_block_3d(outputs, number_of_filters)
# 128, 64
outputs = Concatenate(axis=channels_axis)(
[encoding_convolution_layers[3], outputs])
outputs = feature_block_3d(outputs, number_of_filters)
number_of_filters = int(number_of_filters / 2)
outputs = upsample_block_3d(outputs, number_of_filters)
# 64, 32
outputs = Concatenate(axis=channels_axis)(
[encoding_convolution_layers[2], outputs])
feature64 = feature_block_3d(outputs, number_of_filters)
number_of_filters = int(number_of_filters / 2)
outputs = upsample_block_3d(feature64, number_of_filters)
back64 = None
if do_first_network == True:
back64 = convB_3d_layer(feature64, 1, 1)
else:
back64 = Conv3D(filters=1, kernel_size=1,
data_format=data_format)(feature64)
back64 = UpSampling3D(data_format=data_format)(back64)
# 32, 16
outputs = Concatenate(axis=channels_axis)(
[encoding_convolution_layers[1], outputs])
feature32 = feature_block_3d(outputs, number_of_filters)
number_of_filters = int(number_of_filters / 2)
outputs = upsample_block_3d(feature32, number_of_filters)
back32 = None
if do_first_network == True:
back32 = convB_3d_layer(feature32, 1, 1)
else:
back32 = Conv3D(filters=1, kernel_size=1,
data_format=data_format)(feature32)
back32 = Add()([back64, back32])
back32 = UpSampling3D(data_format=data_format)(back32)
# Final
outputs = Concatenate(axis=channels_axis)(
[encoding_convolution_layers[0], outputs])
outputs = convB_3d_layer(outputs, number_of_filters, 3)
outputs = convB_3d_layer(outputs, number_of_filters, 1)
if do_first_network == True:
outputs = convB_3d_layer(outputs, 1, 1)
else:
outputs = Conv3D(filters=1, kernel_size=1,
data_format=data_format)(outputs)
outputs = Add()([back32, outputs])
outputs = Activation('sigmoid')(outputs)
unet_model = Model(inputs=inputs, outputs=outputs)
return (unet_model)
def bayesian_vnet(
n_classes=1,
input_shape=(256, 256, 256, 1),
kernel_size=3,
prior_fn=prior_fn_for_bayesian(),
kernel_posterior_fn=default_mean_field_normal_fn(),
kld=None,
activation="relu",
padding="SAME",
):
inputs = Input(input_shape)
conv1, pool1 = down_stage(inputs,
16,
kernel_size=kernel_size,
activation=activation,
padding=padding)
conv2, pool2 = down_stage(pool1,
32,
kernel_size=kernel_size,
activation=activation,
padding=padding)
conv3, pool3 = down_stage(pool2,
64,
kernel_size=kernel_size,
activation=activation,
padding=padding)
conv4, _ = down_stage(pool3,
128,
kernel_size=kernel_size,
activation=activation,
padding=padding)
conv5 = up_stage(
conv4,
conv3,
64,
prior_fn,
kernel_posterior_fn,
kld,
kernel_size=kernel_size,
activation=activation,
padding=padding,
)
conv6 = up_stage(
conv5,
conv2,
32,
prior_fn,
kernel_posterior_fn,
kld,
kernel_size=kernel_size,
activation=activation,
padding=padding,
)
conv7 = up_stage(
conv6,
conv1,
16,
prior_fn,
kernel_posterior_fn,
kld,
kernel_size=kernel_size,
activation=activation,
padding=padding,
)
conv8 = end_stage(
conv7,
prior_fn,
kernel_posterior_fn,
kld,
n_classes=n_classes,
kernel_size=kernel_size,
activation=activation,
padding=padding,
)
return Model(inputs=inputs, outputs=conv8)
