def build_model(self,
input_size=(256, 256, 1),
n_layers=4,
n_filters=64,
use_batch_norm=False):
inputs = Input(input_size)
net = inputs
down_layers = []
for _ in range(n_layers):
net = self.conv_block(net, n_filters, use_batch_norm)
print(net.get_shape())
down_layers.append(net)
net = MaxPooling2D((2, 2), strides=2)(net)
n_filters *= 2
net = Dropout(0.5)(net)
net = self.conv_block(net, n_filters, use_batch_norm)
print(net.get_shape())
for conv in reversed(down_layers):
n_filters //= 2
net = self.upsample_block(net, n_filters, (2, 2), use_batch_norm)
print(net.get_shape())
net = concatenate([conv, net], axis=3)
net = self.conv_block(net, n_filters, use_batch_norm)
net = Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(net)
net = Conv2D(1, 1, activation='sigmoid')(net)
self.model = Model(inputs=inputs, outputs=net)
self.model.summary()
def _build(self):
inputs = Input(self.input_shape)
x = self.input_block()(inputs)
x = self.basic_block(x.get_shape().as_list()[3], 256)(x)
x = self.convolution_block(x.get_shape().as_list()[3], 256, 2)(x)
x = Dropout(self.dropout)(x)
x = conv2D_batchnorm(256, [4, 1])(x)
x = Dropout(self.dropout)(x)
classes = conv2D_batchnorm(self.num_classes, [1, 13], padding="same")(x)
pattern = Flatten()(classes)
pattern = Dense(self.pattern_size)(pattern)
width = int(x.get_shape()[2])
pattern = tf.reshape(pattern, (-1, 1, 1, self.pattern_size))
pattern = tf.tile(pattern, [1, 1, width, 1])
x = Concatenate()([classes, pattern])
x = conv2D_batchnorm(self.num_classes, [1, 1], padding="same")(x)
x = tf.squeeze(x, [1])
outs = Softmax()(x)
return Model(inputs=inputs, outputs=outs)
def make_network(self, amino_acid_encoding, peptide_max_length,
n_flank_length, c_flank_length, flanking_averages,
convolutional_filters, convolutional_kernel_size,
convolutional_activation, convolutional_kernel_l1_l2,
dropout_rate, post_convolutional_dense_layer_sizes):
"""
Helper function to make a keras network given hyperparameters.
"""
# We import keras here to avoid tensorflow debug output, etc. unless we
# are actually about to use Keras.
configure_tensorflow()
from tensorflow.keras.layers import (Input, Dense, Dropout,
Concatenate, Conv1D, Lambda)
from tensorflow.keras.models import Model
from tensorflow.keras import regularizers, initializers
model_inputs = {}
empty_x_dict = self.network_input(FlankingEncoding([], [], []))
sequence_dims = empty_x_dict['sequence'].shape[1:]
numpy.testing.assert_equal(
sequence_dims[0],
peptide_max_length + n_flank_length + c_flank_length)
model_inputs['sequence'] = Input(shape=sequence_dims,
dtype='float32',
name='sequence')
model_inputs['peptide_length'] = Input(shape=(1, ),
dtype='int32',
name='peptide_length')
current_layer = model_inputs['sequence']
current_layer = Conv1D(
filters=convolutional_filters,
kernel_size=convolutional_kernel_size,
kernel_regularizer=regularizers.l1_l2(*convolutional_kernel_l1_l2),
padding="same",
activation=convolutional_activation,
name="conv1")(current_layer)
if dropout_rate > 0:
current_layer = Dropout(
name="conv1_dropout",
rate=dropout_rate,
noise_shape=(None, 1, int(
current_layer.get_shape()[-1])))(current_layer)
convolutional_result = current_layer
outputs_for_final_dense = []
for flank in ["n_flank", "c_flank"]:
current_layer = convolutional_result
for (i, size) in enumerate(
list(post_convolutional_dense_layer_sizes) + [1]):
current_layer = Conv1D(
name="%s_post_%d" % (flank, i),
filters=size,
kernel_size=1,
kernel_regularizer=regularizers.l1_l2(
*convolutional_kernel_l1_l2),
activation=("tanh" if size == 1 else
convolutional_activation))(current_layer)
single_output_result = current_layer
dense_flank = None
if flank == "n_flank":
def cleavage_extractor(x):
return x[:, n_flank_length]
single_output_at_cleavage_position = Lambda(
cleavage_extractor,
name="%s_cleaved" % flank)(single_output_result)
def max_pool_over_peptide_extractor(lst):
import tensorflow as tf
(x, peptide_length) = lst
# We generate a per-sample mask that is 1 for all peptide
# positions except the first position, and 0 for all other
# positions (i.e. n flank, c flank, and the first peptide
# position).
starts = n_flank_length + 1
limits = n_flank_length + peptide_length
row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0)
mask = tf.logical_and(tf.greater_equal(row, starts),
tf.less(row, limits))
# We are assuming that x >= -1. The final activation in the
# previous layer should be a function that satisfies this
# (e.g. sigmoid, tanh, relu).
max_value = tf.reduce_max(
(x + 1) *
tf.expand_dims(tf.cast(mask, tf.float32), axis=-1),
axis=1) - 1
# We flip the sign so that initializing the final dense
# layer weights to 1s is reasonable.
return -1 * max_value
max_over_peptide = Lambda(max_pool_over_peptide_extractor,
name="%s_internal_cleaved" % flank)([
single_output_result,
model_inputs['peptide_length']
])
def flanking_extractor(lst):
import tensorflow as tf
(x, peptide_length) = lst
# mask is 1 for n_flank positions and 0 elsewhere.
starts = 0
limits = n_flank_length
row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0)
mask = tf.logical_and(tf.greater_equal(row, starts),
tf.less(row, limits))
# We are assuming that x >= -1. The final activation in the
# previous layer should be a function that satisfies this
# (e.g. sigmoid, tanh, relu).
average_value = tf.reduce_mean(
(x + 1) *
tf.expand_dims(tf.cast(mask, tf.float32), axis=-1),
axis=1) - 1
return average_value
if flanking_averages and n_flank_length > 0:
# Also include average pooled of flanking sequences
pooled_flank = Lambda(flanking_extractor,
name="%s_extracted" % flank)([
convolutional_result,
model_inputs['peptide_length']
])
dense_flank = Dense(1,
activation="tanh",
name="%s_avg_dense" %
flank)(pooled_flank)
else:
assert flank == "c_flank"
def cleavage_extractor(lst):
import tensorflow as tf
(x, peptide_length) = lst
indexer = peptide_length + n_flank_length - 1
result = tf.squeeze(
tf.gather(x, indexer, batch_dims=1, axis=1), -1)
return result
single_output_at_cleavage_position = Lambda(
cleavage_extractor, name="%s_cleaved" % flank)(
[single_output_result, model_inputs['peptide_length']])
def max_pool_over_peptide_extractor(lst):
import tensorflow as tf
(x, peptide_length) = lst
# We generate a per-sample mask that is 1 for all peptide
# positions except the last position, and 0 for all other
# positions (i.e. n flank, c flank, and the last peptide
# position).
starts = n_flank_length
limits = n_flank_length + peptide_length - 1
row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0)
mask = tf.logical_and(tf.greater_equal(row, starts),
tf.less(row, limits))
# We are assuming that x >= -1. The final activation in the
# previous layer should be a function that satisfies this
# (e.g. sigmoid, tanh, relu).
max_value = tf.reduce_max(
(x + 1) *
tf.expand_dims(tf.cast(mask, tf.float32), axis=-1),
axis=1) - 1
# We flip the sign so that initializing the final dense
# layer weights to 1s is reasonable.
return -1 * max_value
max_over_peptide = Lambda(max_pool_over_peptide_extractor,
name="%s_internal_cleaved" % flank)([
single_output_result,
model_inputs['peptide_length']
])
def flanking_extractor(lst):
import tensorflow as tf
(x, peptide_length) = lst
# mask is 1 for c_flank positions and 0 elsewhere.
starts = n_flank_length + peptide_length
limits = n_flank_length + peptide_length + c_flank_length
row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0)
mask = tf.logical_and(tf.greater_equal(row, starts),
tf.less(row, limits))
# We are assuming that x >= -1. The final activation in the
# previous layer should be a function that satisfies this
# (e.g. sigmoid, tanh, relu).
average_value = tf.reduce_mean(
(x + 1) *
tf.expand_dims(tf.cast(mask, tf.float32), axis=-1),
axis=1) - 1
return average_value
if flanking_averages and c_flank_length > 0:
# Also include average pooled of flanking sequences
pooled_flank = Lambda(flanking_extractor,
name="%s_extracted" % flank)([
convolutional_result,
model_inputs['peptide_length']
])
dense_flank = Dense(1,
activation="tanh",
name="%s_avg_dense" %
flank)(pooled_flank)
outputs_for_final_dense.append(single_output_at_cleavage_position)
outputs_for_final_dense.append(max_over_peptide)
if dense_flank is not None:
outputs_for_final_dense.append(dense_flank)
if len(outputs_for_final_dense) == 1:
(current_layer, ) = outputs_for_final_dense
else:
current_layer = Concatenate(name="final")(outputs_for_final_dense)
output = Dense(
1,
activation="sigmoid",
name="output",
kernel_initializer=initializers.Ones(),
)(current_layer)
model = Model(
inputs=[model_inputs[name] for name in sorted(model_inputs)],
outputs=[output],
name="predictor")
return model
def __init__(self,
sess,
ob_space,
action_space,
nbatch,
nsteps,
reuse=False):
# This will use to initialize our kernels
gain = np.sqrt(2)
self.tokenizer = Tokenizer(num_words=5000)
# Based on the action space, will select what probability distribution type
# we will use to distribute action in our stochastic policy (in our case DiagGaussianPdType
# aka Diagonal Gaussian, 3D normal distribution)
self.pdtype = make_pdtype(action_space)
song_text_shape = (None, 200)
category_embedding_shape = (None, 1)
embeddings = []
girl_1_inputs_ = tf.placeholder(tf.float32,
category_embedding_shape,
name="girl_1_inputs_")
girl_1_inputs_keras = tf.keras.layers.Input(tensor=girl_1_inputs_)
embedding_size = EXTRA_SMALL_EMBEDDINGS_DIM
embedding = Embedding(EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)(girl_1_inputs_keras)
embeddings.append(Reshape(target_shape=(embedding_size, ))(embedding))
girl_2_inputs_ = tf.placeholder(tf.float32,
category_embedding_shape,
name="girl_2_inputs_")
girl_2_inputs_keras = tf.keras.layers.Input(tensor=girl_2_inputs_)
embedding_size = EXTRA_SMALL_EMBEDDINGS_DIM
embedding = Embedding(EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)(girl_2_inputs_keras)
embeddings.append(Reshape(target_shape=(embedding_size, ))(embedding))
girl_3_inputs_ = tf.placeholder(tf.float32,
category_embedding_shape,
name="girl_3_inputs_")
girl_3_inputs_keras = tf.keras.layers.Input(tensor=girl_3_inputs_)
embedding_size = EXTRA_SMALL_EMBEDDINGS_DIM
embedding = Embedding(EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)(girl_3_inputs_keras)
embeddings.append(Reshape(target_shape=(embedding_size, ))(embedding))
girl_4_inputs_ = tf.placeholder(tf.float32,
category_embedding_shape,
name="girl_4_inputs_")
girl_4_inputs_keras = tf.keras.layers.Input(tensor=girl_4_inputs_)
embedding_size = EXTRA_SMALL_EMBEDDINGS_DIM
embedding = Embedding(EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)(girl_4_inputs_keras)
embeddings.append(Reshape(target_shape=(embedding_size, ))(embedding))
current_girl_inputs_ = tf.placeholder(tf.float32,
category_embedding_shape,
name="current_girl_inputs_")
current_girl_inputs_keras = tf.keras.layers.Input(
tensor=current_girl_inputs_)
embedding_size = EXTRA_SMALL_EMBEDDINGS_DIM
embedding = Embedding(EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)(current_girl_inputs_keras)
embeddings.append(Reshape(target_shape=(embedding_size, ))(embedding))
# Create the input placeholder
non_category_data_input_ = tf.placeholder(
tf.float32, (None, GUMBALL_FIELD_REMAINDER),
name="non_category_data_input")
combined_inputs_ = tf.placeholder(
tf.float32, (None, ob_space.shape[1] + MM_EMBEDDINGS_DIM * 2),
name="combined_input")
text_inputs_ = tf.placeholder(tf.float32,
song_text_shape,
name="text_input")
available_moves = tf.placeholder(tf.float32, [None, action_space.n],
name="availableActions")
"""
Build the model
Embedding
LSTM
3 FC for spatial dependiencies
1 common FC
1 FC for policy (actor)
1 FC for value (critic)
"""
with tf.variable_scope('model', reuse=reuse):
# text reading LSTM
# lt_layer = lstm_layer()
text_inputs_keras = tf.keras.layers.Input(tensor=text_inputs_)
text_out = lstm_layer(text_inputs_keras)
shape = text_out.get_shape().as_list()[1:] # a list: [None, 9, 2]
dim = np.prod(shape) # dim = prod(9,2) = 18
print('text_flatten before reshape', text_out.shape)
text_flatten = tf.reshape(text_out, [1, -1]) # -1 means "all"
print('embeds', len(embeddings))
merged = Concatenate(axis=-1)(embeddings)
# This returns a tensor
non_category_data_input_keras = tf.keras.layers.Input(
tensor=non_category_data_input_)
categorical_dense = tf.keras.layers.Dense(
512, activation='relu')(merged)
categorical_dense = Reshape(
target_shape=(512, ))(categorical_dense)
non_categorical_dense = tf.keras.layers.Dense(
512, activation='relu')(non_category_data_input_keras)
combined_fields = Concatenate(axis=-1)(
[non_categorical_dense, categorical_dense])
#reshape to add dimension?
comb_shape = combined_fields.get_shape()
combined_fields = K.expand_dims(combined_fields, 2)
print('combined_fields expanded dim', combined_fields.get_shape())
conv1 = Conv1D(
100,
10,
activation='relu',
batch_input_shape=(
None, combined_fields.get_shape()[1]))(combined_fields)
# conv1 = Conv1D(100, 10, activation='relu', batch_input_shape=(None, ob_space.shape[1]))(field_inputs_)
conv1 = Conv1D(100, 10, activation='relu')(conv1)
conv1 = MaxPooling1D(3)(conv1)
conv1 = Conv1D(160, 10, activation='relu')(conv1)
conv1 = Conv1D(160, 10, activation='relu')(conv1)
conv1 = GlobalAveragePooling1D()(conv1)
conv1 = Dropout(0.5)(conv1)
print('conv1 before reshape', conv1.get_shape())
print('text_out before flatten', text_out.get_shape())
text_out = Flatten()(text_out)
print('text_out ater flatten', text_out.get_shape())
text_dense = tf.keras.layers.Dense(512,
activation='relu')(text_out)
field_dense = tf.keras.layers.Dense(512, activation='relu')(conv1)
print('text_dense after dense', text_dense.get_shape())
# scaled_image = tf.keras.layers.Lambda(function=lambda tensors: tensors[0] * tensors[1])([image, scale])
# fc_common_dense = Lambda(lambda x:K.concatenate([x[0], x[1]], axis=1))([text_dense, field_dense])
# fc_common_dense = tf.keras.layers.Concatenate(axis=-1)(list([text_dense, field_dense]))
fc_common_dense = tf.keras.layers.Concatenate(axis=-1)(list(
[text_dense, field_dense]))
fc_common_dense = tf.keras.layers.Dense(
512, activation='relu')(fc_common_dense)
#available_moves takes form [0, 0, -inf, 0, -inf...], 0 if action is available, -inf if not.
fc_act = tf.keras.layers.Dense(256,
activation='relu')(fc_common_dense)
# self.pi = tf.keras.layers.Dense(action_space.n, activation='relu')(fc_act)
self.pi = fc(fc_act, 'pi', action_space.n, init_scale=0.01)
# Calculate the v(s)
h3 = tf.keras.layers.Dense(256, activation='relu')(fc_common_dense)
fc_vf = tf.keras.layers.Dense(1, activation=None)(h3)[:, 0]
# vf = fc_layer(fc_3, 1, activation_fn=None)[:,0]
# vf = fc_layer(fc_common_dense, 1, activation_fn=None)[:,0]
self.initial_state = None
"""
# Take an action in the action distribution (remember we are in a situation
# of stochastic policy so we don't always take the action with the highest probability
# for instance if we have 2 actions 0.7 and 0.3 we have 30% channce to take the second)
a0 = self.pd.sample()
# Calculate the neg log of our probability
neglogp0 = self.pd.neglogp(a0)
"""
# perform calculations using available moves lists
availPi = tf.add(self.pi, available_moves)
def sample():
u = tf.random_uniform(tf.shape(availPi))
return tf.argmax(availPi - tf.log(-tf.log(u)), axis=-1)
a0 = sample()
el0in = tf.exp(availPi -
tf.reduce_max(availPi, axis=-1, keep_dims=True))
z0in = tf.reduce_sum(el0in, axis=-1, keep_dims=True)
p0in = el0in / z0in
onehot = tf.one_hot(a0, availPi.get_shape().as_list()[-1])
neglogp0 = -tf.log(tf.reduce_sum(tf.multiply(p0in, onehot), axis=-1))
# Function use to take a step returns action to take and V(s)
def step(state_in, valid_moves, ob_texts, *_args, **_kwargs):
# return a0, vf, neglogp0
# padd text
# print('ob_text', ob_texts)
for ob_text in ob_texts:
# print('ob_text', ob_text)
self.tokenizer.fit_on_texts([ob_text])
ob_text_input = []
for ob_text in ob_texts:
# print('ob_text', ob_text)
token = self.tokenizer.texts_to_sequences([ob_text])
token = sequence.pad_sequences(
token, maxlen=MM_MAX_SENTENCE_SIZE) # pre_padding with 0
ob_text_input.append(token)
# print('token', token)
# print('token shape', token.shape)
orig_ob_text_input = np.array(ob_text_input)
shape = orig_ob_text_input.shape
# print('ob_text_input shape', shape)
ob_text_input = orig_ob_text_input.reshape(shape[0], shape[2])
# Reshape for conv1
# state_in = np.expand_dims(state_in, axis=2)
input_dict = dict({
text_inputs_: ob_text_input,
available_moves: valid_moves
})
input_dict.update(split_categories_from_state(state_in))
return sess.run([a0, fc_vf, neglogp0], input_dict)
# Function that calculates only the V(s)
def value(state_in, valid_moves, ob_texts, *_args, **_kwargs):
for ob_text in ob_texts:
# print('ob_text', ob_text)
self.tokenizer.fit_on_texts([ob_text])
ob_text_input = []
for ob_text in ob_texts:
# print('ob_text', ob_text)
token = self.tokenizer.texts_to_sequences([ob_text])
token = sequence.pad_sequences(
token, maxlen=MM_MAX_SENTENCE_SIZE) # pre_padding with 0
ob_text_input.append(token)
# print('token', token)
# print('token shape', token.shape)
ob_text_input = np.array(ob_text_input)
shape = ob_text_input.shape
# print('ob_text_input shape', shape)
ob_text_input = ob_text_input.reshape(shape[0], shape[2])
# Reshape for conv1
# state_in = np.expand_dims(state_in, axis=2)
input_dict = dict({
text_inputs_: ob_text_input,
available_moves: valid_moves
})
input_dict.update(split_categories_from_state(state_in))
return sess.run(fc_vf, input_dict)
# return sess.run(vf, {field_inputs_:state_in, text_inputs_:ob_text_input, available_moves:valid_moves})
def select_action(state_in, valid_moves, ob_texts, *_args, **_kwargs):
for ob_text in ob_texts:
# print('ob_text', ob_text)
self.tokenizer.fit_on_texts([ob_text])
ob_text_input = []
for ob_text in ob_texts:
# print('ob_text', ob_text)
token = self.tokenizer.texts_to_sequences([ob_text])
token = sequence.pad_sequences(
token, maxlen=MM_MAX_SENTENCE_SIZE) # pre_padding with 0
ob_text_input.append(token)
# print('token', token)
# print('token shape', token.shape)
ob_text_input = np.array(ob_text_input)
shape = ob_text_input.shape
# print('ob_text_input shape', shape)
ob_text_input = ob_text_input.reshape(shape[0], shape[2])
# Reshape for conv1
# state_in = np.expand_dims(state_in, axis=2)
input_dict = dict({
text_inputs_: ob_text_input,
available_moves: valid_moves
})
input_dict.update(split_categories_from_state(state_in))
return sess.run(fc_vf, input_dict)
# return sess.run(vf, {field_inputs_:state_in, text_inputs_:ob_text_input, available_moves:valid_moves})
def split_categories_from_state(obs_datas):
input_mappings = {}
# Initialize buckets
current_girl = np.empty([0, 1], dtype=np.float32)
girl_1 = np.empty([0, 1], dtype=np.float32)
girl_2 = np.empty([0, 1], dtype=np.float32)
girl_3 = np.empty([0, 1], dtype=np.float32)
girl_4 = np.empty([0, 1], dtype=np.float32)
non_category_data = np.empty([0, GUMBALL_FIELD_REMAINDER],
dtype=np.float32)
input_mappings[current_girl_inputs_] = current_girl
input_mappings[girl_1_inputs_] = girl_1
input_mappings[girl_2_inputs_] = girl_2
input_mappings[girl_3_inputs_] = girl_3
input_mappings[girl_4_inputs_] = girl_4
input_mappings[non_category_data_input_] = non_category_data
# Everything above only happens once
for obs_data in obs_datas:
input_mappings[current_girl_inputs_] = np.append(
input_mappings[current_girl_inputs_],
np.array([[obs_data[0]]]),
axis=0)
input_mappings[girl_1_inputs_] = np.append(
input_mappings[girl_1_inputs_],
np.array([[obs_data[1]]]),
axis=0)
input_mappings[girl_2_inputs_] = np.append(
input_mappings[girl_2_inputs_],
np.array([[obs_data[2]]]),
axis=0)
input_mappings[girl_3_inputs_] = np.append(
input_mappings[girl_3_inputs_],
np.array([[obs_data[3]]]),
axis=0)
input_mappings[girl_4_inputs_] = np.append(
input_mappings[girl_4_inputs_],
np.array([[obs_data[4]]]),
axis=0)
# rest of data is numeric observation
rest_details_index = 5
input_mappings[non_category_data_input_] = np.append(
input_mappings[non_category_data_input_],
np.array([obs_data[rest_details_index:]]),
axis=0)
return input_mappings
self.availPi = availPi
self.split_categories_from_state = split_categories_from_state
self.text_inputs_ = text_inputs_
self.available_moves = available_moves
self.vf = fc_vf
# self.fc_vf = fc_vf
self.step = step
self.value = value
self.select_action = select_action
print('this did finish')
def model_b(n_classes=5, use_sub_layer=False, use_rnn=True, verbose=False):
"""Recurrent_Deep_Neural_Networks_for_Real-Time_Sleep
"""
inputLayer = Input(shape=(3000, 1), name='inLayer')
convFine = Conv1D(filters=512,
kernel_size=int(Fs / 2),
strides=int(Fs / 16),
padding='same',
activation='relu',
name='fConv1')(inputLayer)
convFine = MaxPool1D(pool_size=8, strides=8, name='fMaxP1')(convFine)
convFine = Conv1D(filters=512,
kernel_size=8,
padding='same',
activation='relu',
name='fConv2')(convFine)
convFine = Conv1D(filters=512,
kernel_size=8,
padding='same',
activation='relu',
name='fConv3')(convFine)
convFine = Conv1D(filters=512,
kernel_size=8,
padding='same',
activation='relu',
name='fConv4')(convFine)
convFine = Conv1D(filters=512,
kernel_size=8,
padding='same',
activation='relu',
name='fConv5')(convFine)
convFine = MaxPool1D(pool_size=2, strides=4, name='fMaxP2')(convFine)
fineShape = convFine.get_shape()
convFine = Flatten(name='fFlat1')(convFine)
# network to learn coarse features
convCoarse = Conv1D(filters=256,
kernel_size=Fs * 4,
strides=int(Fs / 2),
padding='same',
activation='relu',
name='cConv1')(inputLayer)
convCoarse = MaxPool1D(pool_size=4, strides=4, name='cMaxP1')(convCoarse)
convCoarse = Dropout(rate=0.5, name='cDrop1')(convCoarse)
convCoarse = Conv1D(filters=256,
kernel_size=6,
padding='same',
activation='relu',
name='cConv2')(convCoarse)
convCoarse = Conv1D(filters=256,
kernel_size=6,
padding='same',
activation='relu',
name='cConv3')(convCoarse)
convCoarse = Conv1D(filters=256,
kernel_size=6,
padding='same',
activation='relu',
name='cConv4')(convCoarse)
convCoarse = Conv1D(filters=256,
kernel_size=6,
padding='same',
activation='relu',
name='cConv5')(convCoarse)
convCoarse = MaxPool1D(pool_size=2, strides=2, name='cMaxP2')(convCoarse)
coarseShape = convCoarse.get_shape()
convCoarse = Flatten(name='cFlat1')(convCoarse)
# concatenate coarse and fine cnns
mergeLayer = concatenate([convFine, convCoarse], name='merge_1')
outLayer = Dropout(rate=0.5, name='mDrop1')(mergeLayer)
outLayer = Reshape((1, outLayer.get_shape()[1]), name='reshape1')(outLayer)
outLayer = LSTM(64, return_sequences=True)(outLayer)
outLayer = LSTM(64, return_sequences=False)(outLayer)
# Classify
outLayer = Dense(n_classes, activation='softmax',
name='outLayer')(outLayer)
model = Model(inputLayer, outLayer)
optimizer = Adam(lr=1e-4)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['acc'])
if verbose:
model.summary()
return model
class ZeepSenseEasy():
def __init__(self):
gpus = tf.config.experimental.list_physical_devices('GPU')
#gpus = None
if gpus:
# Restrict TensorFlow to only allocate 1GB of memory on the first GPU
try:
tf.config.experimental.set_virtual_device_configuration(
gpus[0],
[tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4096)])
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
except RuntimeError as e:
# Virtual devices must be set before GPUs have been initialized
print(e)
self.single_img_length = 200
self.single_input = keras.Input(shape=(2*self.single_img_length, 10 * self.single_img_length, 3), \
name="Input")
self.acce_input = self.single_input[:, 0:self.single_img_length, :, :]
self.gyro_input = self.single_input[:, self.single_img_length:, :, :]
self.acce_input = Reshape((10, self.single_img_length, self.single_img_length, 3), name="Input_acce")(
self.acce_input)
self.gyro_input = Reshape((10, self.single_img_length, self.single_img_length, 3), name="Input_gyro")(
self.gyro_input)
acc_input_dim = self.acce_input.get_shape()
print(acc_input_dim)
gyro_input_dim = self.gyro_input.get_shape()
print(gyro_input_dim)
self.conv1a = Conv3D(64, (1, 5, 5), (1, 3, 3),
name="conv_acce_1",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.acce_input)
# self.conv1a = self.acce_input
# self.conv1a = BatchNormalization(name="conv_acce_1_batchnorm")(self.conv1a)
self.conv1a = Activation("relu", name="conv_acce_1_relu")(self.conv1a)
self.conv1a = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1a.shape[-1]], name="conv_acce_1_dropout")(
self.conv1a)
self.conv1a = AveragePooling3D((1, 2, 2),name="conv_acce_1_pool")(self.conv1a)
acc_conv1a_dim = self.conv1a.get_shape()
print(acc_conv1a_dim)
self.conv2a = Conv3D(64, (1, 3, 3), (1, 1, 1),
name="conv_acce_2",
padding="SAME",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.conv1a)
# self.conv2a = BatchNormalization(name="conv_acce_2_batchnorm")(self.conv2a)
self.conv2a = Activation("relu", name="conv_acce_2_relu")(self.conv2a)
self.conv2a = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1a.shape[-1]], name="conv_acce_2_dropout")(
self.conv2a)
self.conv2a = AveragePooling3D((1, 2, 2), name="conv_acce_2_pool")(self.conv1a)
# self.conv3a = Conv3D(64, (1, 3, 3), (1, 2, 2),
# name="conv_acce_3",
# kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
# )(self.conv2a)
# # self.conv3a = BatchNormalization(name="conv_acce_3_batchnorm")(self.conv3a)
# self.conv3a = Activation("relu", name="conv_acce_3_relu")(self.conv3a)
# self.conv3a = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, 64], name="conv_acce_3_dropout")(
# self.conv3a)
acc_conv2a_dim = self.conv2a.get_shape()
print(acc_conv2a_dim)
self.acce_output = Reshape((10, 1, 16*16, self.conv1a.shape[-1]), name="output_acce")(self.conv2a)
# self.acce_output = self.conv3a
# self.acce_output = AveragePooling3D((1,3,3),(1,2,2))(self.conv3a)
# self.acce_output = Reshape((10,1,10*10,64),name="output_acce")(self.acce_output)
# **********************************************************************************************
acc_output_dim = self.acce_output.get_shape()
print(acc_output_dim)
self.conv1g = Conv3D(64, (1, 5, 5), (1, 3, 3),
name="conv_gyro_1",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.gyro_input)
# self.conv1g = self.gyro_input
# self.conv1g = BatchNormalization(name="conv_gyro_1_batchnorm")(self.conv1g)
self.conv1g = Activation("relu", name="conv_gyro_1_relu")(self.conv1g)
self.conv1g = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1g.shape[-1]], name="conv_gyro_1_dropout")(
self.conv1g)
self.conv1g = AveragePooling3D((1, 2, 2), name="conv_gyro_1_pool")(self.conv1g)
gyro_conv1g_dim = self.conv1g.get_shape()
print(gyro_conv1g_dim)
self.conv2g = Conv3D(64, (1, 3, 3), (1, 1, 1),
name="conv_gyro_2",
padding="SAME",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.conv1g)
# self.conv2g = BatchNormalization(name="conv_gyro_2_batchnorm")(self.conv2g)
self.conv2g = Activation("relu", name="conv_gyro_2_relu")(self.conv2g)
self.conv2g = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv2g.shape[-1]],
name="conv_gyro_2_dropout")(self.conv2g)
self.conv2g = AveragePooling3D((1, 2, 2), name="conv_gyro_2_pool")(self.conv1g)
# self.conv3g = Conv3D(64, (1, 3, 3), (1, 2, 2),
# name="conv_gyro_3",
# kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
# )(self.conv2g)
# # self.conv3g = BatchNormalization(name="conv_gyro_3_batchnorm")(self.conv3g)
# self.conv3g = Activation("relu", name="conv_gyro_3_relu")(self.conv3g)
# self.conv3g = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, 64], name="conv_gyro_3_dropout")(
# self.conv3g)
gyro_conv2g_dim = self.conv2g.get_shape()
print(gyro_conv2g_dim)
self.gyro_output = Reshape((10, 1, 16*16, self.conv1g.shape[-1]), name="output_gyro")(self.conv2g)
# self.gyro_output = self.conv3g
# self.gyro_output = AveragePooling3D((1,3,3),(1,2,2))(self.conv3g)
# self.gyro_output = Reshape((10,1,10*10,64),name="output_gyro")(self.gyro_output)
# ***********************************************************************************************************************
gyro_output_dim = self.gyro_output.get_shape()
print(gyro_output_dim)
#============================no attention===========================================
self.merge_noattention = concatenate([self.acce_output,self.gyro_output],axis = -3, name = "input_merge")
self.merge_input = self.merge_noattention
#==============================attention====================================================
#self.merge_input = concatenate([self.acce_output, self.gyro_output], axis=-3, name="Input_merge")
#self.merge_attention_input = Reshape((10,2,self.merge_input.shape[-2]*self.merge_input.shape[-1]))(self.merge_input)
#self.merge_attention = tf.matmul(self.merge_attention_input,self.merge_attention_input,transpose_b=True)
#self.merge_attention = tf.matmul(tf.ones((1,2),dtype=tf.float32),self.merge_attention)
#softmax_layer = Softmax(axis = -1)
#self.merge_attention = softmax_layer(self.merge_attention)
#merge_attention_dim = self.merge_attention.get_shape()
#print("merge_attention_dim")
#print(merge_attention_dim)
#self.merge_attention_output = tf.matmul(self.merge_attention,self.merge_attention_input)
#self.merge_attention_output = Reshape((10,16,16,64))(self.merge_attention_output)
#merge_attention_output_dim = self.merge_attention_output.get_shape()
#print("merge_attention_output_dim")
#print(merge_attention_output_dim)
#self.merge_input = self.merge_attention_output
#===========================================================================================
self.conv1 = Conv3D(64, kernel_size=(1,2,5),
name='conv_merge_1',
strides=( 1, 1,1),
# padding='SAME',
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.merge_input)
merge_conv1_dim = self.conv1.get_shape()
print("merge_conv1_dim")
print(merge_conv1_dim)
# self.conv1 = BatchNormalization(name="conv_merge_1_batchnorm")(self.conv1)
self.conv1 = Activation("relu", name="conv_merge_1_relu")(self.conv1)
self.conv1 = Dropout(0.2, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1.shape[-1]],
name="conv_merge_1_dropout")(self.conv1)
self.conv1 = AveragePooling3D((1, 1,3))(self.conv1)
# self.conv2 = Conv3D(64, kernel_size=(1, 1, 5),
# name="conv_merge_2",
# strides=(1, 1, 3),
# # padding='SAME',
# kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
# )(self.conv1)
# # self.conv2 = BatchNormalization(name="conv_merge_2_batchnorm")(self.conv2)
# self.conv2 = Activation("relu", name="conv_merge_2_relu")(self.conv2)
# self.conv2 = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, 64], name="conv_merge_2_dropout")(
# self.conv2)
self.conv_output = self.conv1
self.rnn_input = Reshape((10, self.conv_output.shape[-1] * 84), name="Output_merge")(self.conv_output)
self.rnn = LSTM(120, return_sequences=True, name="RNN_1")(self.rnn_input)
self.sum_rnn_out = tf.reduce_sum(self.rnn, axis=1, keep_dims=False)
self.rnn = LSTM(20, name="RNN_2")(self.rnn)
self.rnn_output = Dense(6, 'softmax', name="Softmax")(self.rnn)
self.model = keras.Model(
inputs=self.single_input,
outputs=self.rnn_output)
def train(self, file_dir, val_dir, epochs, save_dir=None):
self.model.compile(optimizer=keras.optimizers.Adam(),
loss=keras.losses.SparseCategoricalCrossentropy(),
metrics=['acc'])
data_init = Tfdata(file_dir)
data = data_init.acquire_data()
val_data_init = Tfdata(val_dir)
val_data = val_data_init.acquire_data(False)
print("Training results:")
self.history = LossHistory()
self.model.fit(data,
epochs=epochs,
verbose=2,
validation_data=val_data,
callbacks=[self.history])
# self.model.save(save_dir)
def evaluate(self, val_dir):
val_data_init = Tfdata(val_dir)
val_data = val_data_init.acquire_data(False)
print("y_true in evaluation")
print(val_data_init.raw_labels)
Y_pred = self.model.predict(val_data)
y_pred = np.argmax(Y_pred, axis=1)
y_true = val_data_init.raw_labels[0:len(y_pred) // BATCH_SIZE * BATCH_SIZE]
print("y_ture in confusion matrix.")
print(y_true)
output_matrix = confusion_matrix(y_true, y_pred) / (len(y_true) / 6)
plt.figure(2)
class_names = GESTURES
tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names, rotation=30)
plt.yticks(tick_marks, class_names)
plt.imshow(output_matrix, cmap=plt.cm.Reds)
plt.xlabel("predicted labels", fontsize='large')
plt.ylabel("true labels", fontsize="large")
plt.title("Normalized Confusion Matrix")
plt.colorbar(orientation='vertical')
plt.savefig("GAF4ZS/f200/confusion_matrix100.jpg", bbox_inches='tight')
plt.tight_layout()
plt.close(2)
f1 = metrics.f1_score(y_true, y_pred, average='micro')
f2 = metrics.f1_score(y_true, y_pred, average='macro')
print('micro f1 score: {}, macro f1 score:{}'.format(f1,f2))
