def f(notes, beat, style):
time_steps = int(notes.get_shape()[1])
# TODO: Experiment with when to apply conv
note_octave = TimeDistributed(Conv1D(OCTAVE_UNITS, 2 * OCTAVE, padding='same'))(notes)
note_octave = Activation('tanh')(note_octave)
note_octave = Dropout(dropout)(note_octave)
# Create features for every single note.
note_features = Concatenate()([
Lambda(pitch_pos_in_f(time_steps))(notes),
Lambda(pitch_class_in_f(time_steps))(notes),
Lambda(pitch_bins_f(time_steps))(notes),
note_octave,
TimeDistributed(RepeatVector(NUM_NOTES))(beat)
])
x = note_features
# [batch, notes, time, features]
x = Permute((2, 1, 3))(x)
# Apply LSTMs
for l in range(TIME_AXIS_LAYERS):
# Integrate style
style_proj = Dense(int(x.get_shape()[3]))(style)
style_proj = TimeDistributed(RepeatVector(NUM_NOTES))(style_proj)
style_proj = Activation('tanh')(style_proj)
style_proj = Dropout(dropout)(style_proj)
style_proj = Permute((2, 1, 3))(style_proj)
x = Add()([x, style_proj])
x = TimeDistributed(LSTM(TIME_AXIS_UNITS, return_sequences=True))(x)
x = Dropout(dropout)(x)
# [batch, time, notes, features]
return Permute((2, 1, 3))(x)
def get_untrained_model(encoder_dropout=0,
decoder_dropout=0,
input_dropout=0,
reg_W=0,
reg_B=0,
reg_act=0,
LSTM_size=256,
dense_size=100,
maxpooling=True,
data_dim=300,
max_len=22):
'''
Creates a neural network with the specified conditions.
params:
encoder_dropout: dropout rate for LSTM encoders (NOT dropout for LSTM internal gates)
decoder_dropout: dropout rate for decoder
reg_W: lambda value for weight regularization
reg_b: lambda value for bias regularization
reg_act: lambda value for activation regularization
LSTM_size: number of units in the LSTM layers
maxpooling: pool over LSTM output at each timestep, or just take the output from the final LSTM timestep
data_dim: dimension of the input data
max_len: maximum length of an input sequence (this should be found based on the training data)
nb_classes: number of classes present in the training data
'''
# create regularization objects if needed
if reg_W != 0:
W_reg = l2(reg_W)
else:
W_reg = None
if reg_B != 0:
B_reg = l2(reg_B)
else:
B_reg = None
if reg_act != 0:
act_reg = activity_l2(reg_act)
else:
act_reg = None
# encode the first entity
encoder_L = Sequential()
encoder_L.add(Dropout(input_dropout, input_shape=(data_dim, max_len)))
encoder_L.add(Permute((2, 1)))
print "Set encoder dropout ", encoder_dropout
# with maxpooling
if maxpooling:
encoder_L.add(
LSTM(LSTM_size,
return_sequences=True,
inner_activation="hard_sigmoid"))
if encoder_dropout != 0:
encoder_L.add(TimeDistributed(Dropout(encoder_dropout)))
encoder_L.add(Permute((2, 1)))
encoder_L.add(MaxPooling1D(pool_length=LSTM_size))
encoder_L.add(Flatten())
# without maxpooling
else:
encoder_L.add(Masking(mask_value=0.))
encoder_L.add(
LSTM(LSTM_size,
return_sequences=False,
inner_activation="hard_sigmoid"))
if encoder_dropout != 0:
encoder_L.add(Dropout(encoder_dropout))
# encode the second entity
encoder_R = Sequential()
encoder_R.add(Dropout(input_dropout, input_shape=(data_dim, max_len)))
encoder_R.add(Permute((2, 1)))
# with maxpooling
if maxpooling:
encoder_R.add(
LSTM(LSTM_size,
return_sequences=True,
inner_activation="hard_sigmoid"))
if encoder_dropout != 0:
encoder_R.add(TimeDistributed(Dropout(encoder_dropout)))
encoder_R.add(Permute((2, 1)))
encoder_R.add(MaxPooling1D(pool_length=LSTM_size))
encoder_R.add(Flatten())
else:
# without maxpooling
encoder_R.add(Masking(mask_value=0.))
encoder_R.add(
LSTM(LSTM_size,
return_sequences=False,
inner_activation="hard_sigmoid"))
if encoder_dropout != 0:
encoder_R.add(Dropout(encoder_dropout))
# combine and classify entities as a single relation
decoder = Sequential()
decoder.add(Merge([encoder_R, encoder_L], mode='concat'))
decoder.add(
Dense(dense_size,
W_regularizer=W_reg,
b_regularizer=B_reg,
activity_regularizer=act_reg,
activation='relu',
W_constraint=maxnorm(4)))
if decoder_dropout != 0:
decoder.add(Dropout(decoder_dropout))
print "Set decoder dropout ", decoder_dropout
decoder.add(
Dense(1,
W_regularizer=None,
b_regularizer=B_reg,
activity_regularizer=act_reg,
activation='sigmoid'))
# compile the final model
# opt = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) # learning rate 0.001 is the default value
opt = SGD(lr=0.01, momentum=0.9, decay=0.0, nesterov=False)
decoder.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return decoder
def _scorer(doc_inputs):
# Pass to another variable not to override doc_inputs in the future
inputs = doc_inputs
if not use_static_matrices:
query, doc = embedding_layer(inputs['query']), embedding_layer(inputs['doc'])
# Cut query dimension
query = Lambda(lambda x: x[:, :self.p['max_query_len'], :])(query)
# Build similarity matrix
if not use_static_matrices:
inputs = {max(ngrams): build_matrix([query, doc])}
if extra_matrix is not None:
inputs['extra_matrix'] = extra_matrix([query, doc])
# Add query IDF vector directly to FF layers
if use_query_idf_config:
doc_qts_scores = [query_idf_score]
else:
doc_qts_scores = []
matrices = [inputs[max(ngrams)]] if extra_matrix is None \
else [inputs[max(ngrams)], inputs['extra_matrix']]
for n in ngrams:
dim_name = get_name(n, n)
for i, matrix in enumerate(matrices):
if i == 1 and n != 1:
continue # dont use convs for extra_matrix
if n == 1:
# No Convolution for 1-gram
re_doc_cov = matrix
elif use_convs:
# Add channel dimension (1st dimension is batch size)
if len(matrix.shape) == 3:
matrix = re_input(matrix)
else:
matrix = matrix
# Pass input by Convolution Layer
re_doc_cov = conv_layers[dim_name](matrix)
if use_masking:
mask = inputs["%s_mask" % max(ngrams)]
# Add channel dimension (1st dimension is batch size)
if len(mask.shape) == 3:
mask = re_input(mask)
re_doc_cov = Multiply()([re_doc_cov, mask])
# Batch normalization after non-linearity
if batch_norm:
re_doc_cov = batch_norms[dim_name](re_doc_cov)
else:
# Don't use conv
continue
if len(re_doc_cov.shape) == 4:
# Reduce channels to 1
if use_maxpooling:
# Pass Conv output through MaxPooling Layer (after permuting) and remove axis=2
re_doc_cov = maxpool_layer[dim_name](Permute((1, 3, 2))(re_doc_cov))
else:
# Pass Conv output through Conv1x1, permute and remove axis=2
re_doc_cov = Permute((1, 3, 2))(conv1x1_layers[dim_name](re_doc_cov))
if top_k != 0:
re_doc_cov = squeeze[dim_name](dropouts[dim_name](re_doc_cov))
else:
re_doc_cov = Permute((1, 3, 2))(dropouts[dim_name](re_doc_cov))
# Batch normalization after non-linearity
if batch_norm:
re_doc_cov = batch_norms1x1[dim_name](re_doc_cov)
else:
re_doc_cov = re_input(re_doc_cov) if top_k == 0 else re_doc_cov
# Get top_k max values for each row in the matrix (K-MaxPooling Layer)
# Or use attention mechanism
if use_context:
ng_signal = pool_top_k_layer_context[dim_name]([re_doc_cov, inputs['context']])
else:
# # Attention mechanism
# ng_signal = Permute((2, 1))(pool_top_k_layer[dim_name]([doc, query, re_doc_cov]))
if top_k != 0:
# K-Maxpooling layer
ng_signal = pool_top_k_layer[dim_name](re_doc_cov)
else:
# Multiple convs
# Pass input by Convolution Layer
re_doc_cov = extra_conv_layers[dim_name](re_doc_cov)
if use_maxpooling:
# Pass Conv output through MaxPooling Layer (after permuting) and remove axis=2
ng_signal = squeeze[dim_name](dropouts[dim_name](extra_maxpool_layer[dim_name](Permute((1, 3, 2))(re_doc_cov))))
else:
# Pass Conv output through Conv1x1, permute and remove axis=2
ng_signal = squeeze[dim_name](dropouts[dim_name](Permute((1, 3, 2))(extra_conv1x1_layers[dim_name](re_doc_cov))))
doc_qts_scores.append(ng_signal)
# Concatenate scores for each query term
if len(doc_qts_scores) == 1:
doc_qts_score = doc_qts_scores[0]
else:
doc_qts_score = Concatenate(axis=2)(doc_qts_scores)
# Permute query positions
if permute_idxs is not None:
doc_qts_score = Lambda(_permute_scores)([doc_qts_score, permute_idxs])
# Get a final score
doc_score = head_layer(doc_qts_score)
return doc_score
def get_model(modelname,axis=None,loss=None):
if modelname == 'Unet':
x = Input((80,80,40,1))
conv0 = block_warp('conv',x,32)
downconv1 = Conv3D(48,kernel_size=3,strides=2,padding='same')(conv0)
downconv1 = BatchNormalization()(downconv1)
downconv1 = Activation('relu')(downconv1)
conv1 = block_warp('conv',downconv1,64)
downconv2 = Conv3D(96, kernel_size=3, strides=2, padding='same')(conv1)
downconv2 = BatchNormalization()(downconv2)
downconv2 = Activation('relu')(downconv2)
conv2 = block_warp('conv',downconv2,128)
deconv1 = block_warp('deconv', conv2,64)
deconv1 = block_warp('conv', concatenate([deconv1,conv1]),64)
deconv2 = block_warp('deconv', deconv1,32)
deconv2 = block_warp('conv', concatenate([deconv2, conv0]),64)
output = Conv3D(filters=1,kernel_size=3,padding='same',activation='tanh')(deconv2)
# output = BatchNormalization()(output)
elif modelname == 'convlstm':
assert axis is not None
if axis == 'x': # x y z
x = Input((80, 80, 40, 1))
elif axis == 'y': # y x z
x = Input((80, 80, 40, 1))
elif axis == 'z': # z y x
x = Input((40, 80, 80, 1))
else:
raise ValueError("convlstm axis error")
conv0 = block_warp('conv',x,32)
downconv1 = Conv3D(48, kernel_size=3, strides=2, padding='same')(conv0)
downconv1 = BatchNormalization()(downconv1)
downconv1 = Activation('relu')(downconv1)
x_z = downconv1
x_x = Permute((3, 2, 1, 4))(downconv1)
x_y = Permute((2, 1, 3, 4))(downconv1)
lstm_z = Bidirectional(ConvLSTM2D(filters=64,padding='same',return_sequences=True,kernel_size=3)
,merge_mode='sum')(x_z)
lstm_y = Bidirectional(ConvLSTM2D(filters=64,padding='same',return_sequences=True,kernel_size=3)
, merge_mode='sum')(x_y)
lstm_y = Permute((2,1,3,4))(lstm_y)
lstm_x = Bidirectional((ConvLSTM2D(filters=64,padding='same',return_sequences=True,kernel_size=3))
, merge_mode='sum')(x_x)
lstm_x = Permute((3,2,1,4))(lstm_x)
deconv1 = block_warp('conv', concatenate([lstm_x,lstm_y,lstm_z]),96)
decoder = Conv3DTranspose(filters=64,kernel_size=3,strides=2,padding='same')(deconv1)
# conv1 = block_warp('conv',conv0,32)
#
# decoder = multiply([conv1,decoder])
output = Conv3D(filters=1,kernel_size=3,activation='tanh',padding='same')(decoder)
else:
raise ValueError("don't have this model")
output = Lambda(norm_layer)(output)
assert loss is not None
# with tf.device('/gpu:3'):
model = Model(inputs=[x],outputs=[output])
# model = multi_gpu_model(model,gpus=4)
model.compile(optimizer=Nadam(lr=0.0003),loss=loss,metrics=[diceMetric])
print(model.summary())
return model
def SenWeightedSum(attentions, representations):
repeated_attentions = RepeatVector(K.int_shape(representations)[-1])(attentions)
repeated_attentions = Permute([2, 1])(repeated_attentions)
aggregated_representation = Multiply()([representations, repeated_attentions])
return aggregated_representation
def FCN():
FCN_CLASSES = 21
#(samples, channels, rows, cols)
input_img = Input(shape=(3, 224, 224))
#(3*224*224)
x = Convolution2D(64, 3, 3, activation='relu',border_mode='same')(input_img)
x = Convolution2D(64, 3, 3, activation='relu',border_mode='same')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#(64*112*112)
x = Convolution2D(128, 3, 3, activation='relu',border_mode='same')(x)
x = Convolution2D(128, 3, 3, activation='relu',border_mode='same')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#(128*56*56)
x = Convolution2D(256, 3, 3, activation='relu',border_mode='same')(x)
x = Convolution2D(256, 3, 3, activation='relu',border_mode='same')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#(256*56*56)
#split layer
p3 = x
p3 = Convolution2D(FCN_CLASSES, 1, 1,activation='relu')(p3)
#(21*28*28)
x = Convolution2D(512, 3, 3, activation='relu',border_mode='same')(x)
x = Convolution2D(512, 3, 3, activation='relu',border_mode='same')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#(512*14*14)
#split layer
p4 = x
p4 = Convolution2D(FCN_CLASSES, 1, 1, activation='relu')(p4)
p4 = Deconvolution2D(FCN_CLASSES, 4, 4,
output_shape=(None, FCN_CLASSES, 30, 30),
subsample=(2, 2),
border_mode='valid')(p4)
p4 = Cropping2D(cropping=((1, 1), (1, 1)))(p4)
#(21*28*28)
x = Convolution2D(512, 3, 3, activation='relu',border_mode='same')(x)
x = Convolution2D(512, 3, 3, activation='relu',border_mode='same')(x)
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#(512*7*7)
p5 = x
p5 = Convolution2D(FCN_CLASSES, 1, 1, activation='relu')(p5)
p5 = Deconvolution2D(FCN_CLASSES, 8, 8,
output_shape=(None, FCN_CLASSES, 32, 32),
subsample=(4, 4),
border_mode='valid')(p5)
p5 = Cropping2D(cropping=((2, 2), (2, 2)))(p5)
#(21*28*28)
# merge scores
merged = merge([p3, p4, p5], mode='sum')
x = Deconvolution2D(FCN_CLASSES, 16, 16,
output_shape=(None, FCN_CLASSES, 232, 232),
subsample=(8, 8),
border_mode='valid')(merged)
x = Cropping2D(cropping=((4, 4), (4, 4)))(x)
x = Reshape((FCN_CLASSES,224*224))(x)
x = Permute((2,1))(x)
out = Activation("softmax")(x)
#(21,224,224)
model = Model(input_img, out)
return model
def cbrnn_dynamic(num_classes, dimx, dimy, acts, **kwargs):
"""
"""
pools = kwargs['kwargs'].get('pools', [])
drops = kwargs['kwargs'].get('drops', [])
bn = kwargs['kwargs'].get('batch_norm', False)
end_dense = kwargs['kwargs'].get('end_dense', {})
last_act = kwargs['kwargs'].get('last_act', 'softmax')
cnn_layers = kwargs['kwargs'].get('cnn_layers', 1)
rnn_layers = kwargs['kwargs'].get('rnn_layers', 1)
rnn_type = kwargs['kwargs'].get('rnn_type', 'LSTM')
rnn_units = kwargs['kwargs'].get('rnn_units', [])
nb_filter = kwargs['kwargs'].get('nb_filter', [])
filter_length = kwargs['kwargs'].get('filter_length', [])
#CNN with biderectional lstm
print("CBRNN")
if not np.all([
len(acts) == cnn_layers,
len(nb_filter) == cnn_layers,
len(filter_length) == cnn_layers
]):
print("Layers Mismatch")
return False
x = Input(shape=(1, dimx, dimy), name='inpx')
inpx = x
for i in range(cnn_layers):
x = Conv2D(filters=nb_filter[i],
kernel_size=filter_length[i],
data_format='channels_first',
padding='same',
activation=acts[i])(x)
if bn:
x = BatchNormalization()(x)
if pools != []:
if pools[i][0] == 'max':
x = MaxPooling2D(pool_size=pools[i][1])(x)
elif pools[i][0] == 'avg':
x = AveragePooling2D(pool_size=pools[i][1])(x)
if drops != []:
x = Dropout(drops[i])(x)
x = Permute((2, 1, 3))(x)
a, b, c, d = kr(x)
x = Reshape((b * d, c))(x)
for i in range(rnn_layers):
#Only last layer can have return_sequences as False
r = False if i == rnn_layers - 1 else True
if rnn_type == 'LSTM':
x = LSTM(rnn_units[i], return_sequences=r)(x)
elif rnn_type == 'GRU':
x = Bidirectional(GRU(rnn_units[i], return_sequences=r))(x)
elif rnn_type == 'bdLSTM':
x = Bidirectional(LSTM(rnn_units[i], return_sequences=r))(x)
elif rnn_type == 'bdGRU':
x = Bidirectional(GRU(rnn_units[i], return_sequences=r))(x)
x = Dropout(0.1)(x)
if end_dense != {}:
x = Dense(end_dense['input_neurons'],
activation=end_dense['activation'],
name='wrap')(x)
try:
x = Dropout(end_dense['dropout'])(x)
except:
pass
main_output = Dense(num_classes, activation=last_act,
name='main_output')(x)
model = Model(inputs=inpx, outputs=main_output)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def cbrnn(dimx, dimy, num_classes, **kwargs):
"""
CNN with biderectional lstm
Parameters
----------
rnn_units : int
default : 32
Number of Units for LSTM layer.
dropout : float
default : 0.1
Dropout used after the Dense Layer.
act1 : str
default : relu
Activation used after 4 Convolution layers.
act2 : str
default : sigmoid
Activation used after Recurrent layer.
act3 : str
default : sigmoid
Activation used after Dense layer.
print_sum : bool
default : False
Print summary if the model
nb_filter : int
default : 100
Number of kernels
filter_length : int, tuple
default : 3
Size of kernels
pool_size : int, tuple
default : (2,2)
Pooling size.
loss
default : categorical_crossentropy
Loss used
optimizer
default : adam
Optimizer used
metrics
default : accuracy
Metrics used.
Returns
-------
CBRNN Model
"""
rnn_units = kwargs['kwargs'].get('rnn_units', 64)
act1 = kwargs['kwargs'].get('act1', 'relu')
act2 = kwargs['kwargs'].get('act2', 'sigmoid')
act3 = kwargs['kwargs'].get('act3', 'sigmoid')
dropout = kwargs['kwargs'].get('dropout', 0.1)
nb_filter = kwargs['kwargs'].get('nb_filter', 100)
filter_length = kwargs['kwargs'].get('filter_length', 5)
pool_size = kwargs['kwargs'].get('pool_size', (2, 2))
print_sum = kwargs['kwargs'].get('print_sum', False)
loss = kwargs['kwargs'].get('loss', 'binary_crossentropy')
optimizer = kwargs['kwargs'].get('optimizer', 'adam')
metrics = kwargs['kwargs'].get('metrics', 'accuracy')
# print("Functional CBRNN")
# print("Activation 1 {} 2 {} 3 {}".format(act1,act2,act3))
# print("Dropout {}".format(dropout))
# print("Kernels {} Size {} Poolsize {}".format(nb_filter,filter_length,pool_size))
# print("Loss {} Optimizer {} Metrics {}".format(loss,optimizer,metrics))
main_input = Input(shape=(1, dimx, dimy))
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1,
use_bias=False)(main_input)
#x1=BatchNormalization()(x)
hx = MaxPooling2D(pool_size=pool_size)(x)
# wrap= Dropout(dropout)(hx)
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1,
use_bias=False)(hx)
#x2=BatchNormalization()(x)
hx = MaxPooling2D(pool_size=pool_size)(x)
# wrap= Dropout(dropout)(hx)
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1,
use_bias=False)(hx)
#x3=BatchNormalization()(x)
hx = MaxPooling2D(pool_size=(2, 2))(x)
# wrap= Dropout(dropout)(hx)
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1,
use_bias=False)(hx)
# x4=BatchNormalization()(x)
hx = MaxPooling2D(pool_size=(1, 1))(x)
wrap = Dropout(dropout)(x)
x = Permute((2, 1, 3))(wrap)
a, b, c, d = kr(x)
x = Reshape((b * d, c))(x)
# x = Reshape((c*d,b))(x)
# w = Bidirectional(LSTM(rnn_units,activation=act2,return_sequences=False))(x)
rnnout = Bidirectional(
LSTM(rnn_units, activation=act2, return_sequences=True))(x)
rnnout_gate = Bidirectional(
LSTM(rnn_units, activation=act3, return_sequences=False))(x)
w = Multiply()([rnnout, rnnout_gate])
wrap = Dropout(dropout)(w)
wrap = Flatten()(wrap)
main_output = Dense(num_classes, activation=act3, name='main_output')(wrap)
model = Model(inputs=main_input, outputs=main_output)
if print_sum:
model.summary()
model.compile(loss=loss, optimizer=optimizer, metrics=[metrics])
return model
env.set_obs_type("Image")
np.random.seed(123)
env.seed(123)
nb_agents = env.n_agents
# List to hold DQN agents for this envs.
agents = []
for idx in range(nb_agents):
nb_actions = env.action_space.spaces[idx].n
# Next, we build our model. We use the same model that was described by Mnih et al. (2015).
input_shape = (WINDOW_LENGTH, ) + INPUT_SHAPE
model = Sequential()
if K.image_dim_ordering() == 'tf':
# (width, height, channels)
model.add(Permute((2, 3, 1), input_shape=input_shape))
elif K.image_dim_ordering() == 'th':
# (channels, width, height)
model.add(Permute((1, 2, 3), input_shape=input_shape))
else:
raise RuntimeError('Unknown image_dim_ordering.')
model.add(Convolution2D(32, 8, 8, subsample=(4, 4)))
model.add(Activation('relu'))
model.add(Convolution2D(64, 4, 4, subsample=(2, 2)))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3, subsample=(1, 1)))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
def cnn_1d_model(hparams, context, utterances):
# Initialize embeddings randomly or with pre-trained vectors
embeddings_W = get_embeddings(hparams)
print("embeddings_W: ", embeddings_W.shape)
# Define embedding layer shared by context and 100 utterances
embedding_layer = Embedding(input_dim=hparams.vocab_size,
output_dim=hparams.embedding_dim,
weights=[embeddings_W],
input_length=hparams.max_seq_len,
mask_zero=False,
trainable=True)
# Context Embedding (Output shape: BATCH_SIZE(?) x LEN_SEQ(160) x EMBEDDING_DIM(300))
context_embedded = embedding_layer(context)
# context_embedded = Masking()(context_embedded)
print("context_embedded: ", context_embedded.shape)
print("context_embedded (history): ", context_embedded._keras_history)
# Utterances Embedding (Output shape: NUM_OPTIONS(100) x BATCH_SIZE(?) x LEN_SEQ(160) x EMBEDDING_DIM(300))
# -> Utterances_embedded: (?, 100, 160, 300)
utterances_embedded = TimeDistributed(
embedding_layer,
input_shape=(hparams.num_utterance_options,
hparams.max_seq_len))(utterances)
print("Utterances_embedded: ", utterances_embedded.shape)
print("Utterances_embedded (history): ",
utterances_embedded._keras_history)
# Define CNN context & utterances encoders
context_max_maps = []
utterances_max_maps = []
for k_s in hparams.kernel_size:
CNN_1D = Conv1D(filters=hparams.num_filters,
kernel_size=k_s,
strides=1,
padding="valid",
activation="relu",
input_shape=(hparams.max_seq_len,
hparams.embedding_dim))
# Output shape: BATCH_SIZE(?) x (LEN_SEQ - k_s + 1) x NUM_FILTERS (for one kernel_size)
context_feature_map = CNN_1D(context_embedded)
# Output shape: BATCH_SIZE(?) x 1 x NUM_FILTERS (for one kernel_size)
context_max_map = MaxPooling1D(pool_size=hparams.max_seq_len - k_s +
1)(context_feature_map)
context_max_maps.append(Flatten()(context_max_map))
CNN_2D = Conv2D(filters=hparams.num_filters,
kernel_size=(1, k_s),
strides=1,
padding="valid",
activation='relu',
input_shape=(hparams.num_utterance_options,
hparams.max_seq_len,
hparams.embedding_dim))
# Output shape: BATCH_SIZE(?) x NUM_UTTERANCES(100) x (LEN_SEQ - k_s + 1) x NUM_FILTERS (for one kernel_size)
utterances_feature_map = CNN_2D(utterances_embedded)
# Output shape: BATCH_SIZE(?) x NUM_UTTERANCES(100) x 1 x NUM_FILTERS (for one kernel_size)
utterances_max_map = MaxPooling2D(
pool_size=(1,
hparams.max_seq_len - k_s + 1))(utterances_feature_map)
# Output shape: BATCH_SIZE(?) x NUM_UTTERANCES(100) x NUM_FILTERS (for one kernel_size)
Flatten_u_by_u = Reshape(
(hparams.num_utterance_options, hparams.num_filters))
utterances_max_maps.append(Flatten_u_by_u(utterances_max_map))
# Output shape: BATCH_SIZE(?) x (NUM_KERNEL_SIZES x NUM_FILTERS)
context_encoded = Concatenate()(context_max_maps)
print("context_encoded: ", context_encoded.shape)
print("context_encoded: ", context_encoded._keras_history)
context_encoded = Dropout(hparams.cnn_drop_rate)(context_encoded)
# Output shape: BATCH_SIZE(?) x NUM_UTTERANCES(100) x (NUM_KERNEL_SIZES x NUM_FILTERS)
# -> BATCH_SIZE(?) x (NUM_KERNEL_SIZES x NUM_FILTERS) x NUM_UTTERANCES(100)
all_utterances_encoded = Concatenate(axis=2)(utterances_max_maps)
all_utterances_encoded = Flatten()(all_utterances_encoded)
all_utterances_encoded = Dropout(
hparams.cnn_drop_rate)(all_utterances_encoded)
all_utterances_encoded = Reshape(
(hparams.num_utterance_options, len(hparams.kernel_size) *
hparams.num_filters))(all_utterances_encoded)
print("all_utterances_encoded: ", all_utterances_encoded.shape)
print("all_utterances_encoded: ", all_utterances_encoded._keras_history)
all_utterances_encoded = Permute((2, 1))(all_utterances_encoded)
# Generate (expected) response from context: C_transpose * M
# (Output shape: BATCH_SIZE(?) x (NUM_KERNEL_SIZES x NUM_FILTERS)
matrix_multiplication_layer = Dense(
len(hparams.kernel_size) * hparams.num_filters,
use_bias=False,
kernel_initializer=keras.initializers.TruncatedNormal(mean=0.0,
stddev=1.0,
seed=None))
generated_response = matrix_multiplication_layer(context_encoded)
print("genearted_response: ", generated_response.shape)
print("history555555: ", generated_response._keras_history)
# (Output shape: BATCH_SIZE(?) x 1 x (NUM_KERNEL_SIZES x NUM_FILTERS)
generated_response = Reshape(
(1,
len(hparams.kernel_size) * hparams.num_filters))(generated_response)
print("genearted_response_expand_dims: ", generated_response.shape)
print("genearted_response_expand_dims: ",
generated_response._keras_history)
# Dot product between generated response and each of 100 utterances(actual response r): C_transpose * M * r
# (Output shape: BATCH_SIZE(?) x EXTRA_DIM(1) x NUM_OPTIONS(100))
batch_matrix_multiplication_layer = Lambda(
lambda x: K.batch_dot(x[0], x[1]))
logits = batch_matrix_multiplication_layer(
[generated_response, all_utterances_encoded])
print("logits: ", logits.shape)
print("logtis: ", logits._keras_history)
### squeeze_layer = Lambda(lambda x: K.squeeze(x, 1))
### logits = squeeze_layer(logits)
# Squeezing logits (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100))
logits = Reshape((hparams.num_utterance_options, ),
input_shape=(1, hparams.num_utterance_options))(logits)
print("logits_squeeze: ", logits.shape)
print("logits_squeeze: ", logits._keras_history)
# Softmax layer for probability of each of Dot products in previous layer
# Softmaxing logits (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100))
probs = Activation('softmax', name='probs')(logits)
print("probs: ", probs.shape)
print("final History: ", probs._keras_history)
# Return probabilities(likelihoods) of each of utterances
# Those will be used to calculate the loss ('sparse_categorical_crossentropy')
return probs
Block) and after that using a dense network that is reshaped. 32x32x3 -> (1x1) conv 32x32x1 -> (3x3)x64 conv(same padding) 32x32x64 -> ---> Attention block will be small dense network with conv and softmax ---> Attention block * last layer ---> sum over 32x32 values and get 64 neurons -> Bigger dense network -> softmax -> prediction """ img_inputs = Input(shape=input_dim) conv1 = Conv2D(1, (1, 1), padding='same', activation='relu')(img_inputs) #1 conv = Conv2D(128, (4, 4), padding='same', activation='relu')(conv1) #2 conv = BatchNormalization()(conv) #Attention 1 y = Conv2D(1, (1, 1))(conv) # 32x32x1 ? y = Permute((3, 2, 1))(y) y = Dense(32, activation='softmax')(y) y = Permute((1, 3, 2))(y) y = Dense(32, activation='softmax')(y) y = Permute((1, 3, 2))(y) #now permute back y = Permute((3, 2, 1))(y) #end attention mult = Multiply()([conv, y]) pooled = MaxPooling2D(pool_size=(2, 2))(mult) pooled = Dropout(0.2)(pooled) conv = Conv2D(128, (3, 3), padding='same', activation='relu')(pooled) conv = BatchNormalization()(conv) #Attention 2 y = Conv2D(1, (1, 1))(conv) # 32x32x1 ? y = Permute((3, 2, 1))(y)
# Next, we build our model.
input_shape = (WINDOW_LENGTH,) + INPUT_SHAPE
# actions = ['NOOP', 'FIRE', 'RIGHT', 'LEFT']
# build constant masking layers
eye = np.eye(nb_actions)
masks = []
for i in range(nb_actions):
mask = np.zeros((nb_actions, nb_actions))
mask[:, i] = eye[:, i]
masks.append(mask)
masks = np.array(masks)
InpLayer = Input(shape=input_shape)
if K.image_dim_ordering() == 'tf':
X = Permute((2, 3, 1))(InpLayer)
elif K.image_dim_ordering() == 'th':
X = Permute((1, 2, 3))(InpLayer)
else:
raise RuntimeError('Unknown image_dim_ordering.')
X = Conv2D(32, 8, strides=4, activation='relu')(X)
X = Conv2D(64, 4, strides=2, activation='relu')(X)
X = Conv2D(64, 3, strides=1, activation='relu')(X)
X = MaxPool2D(2)(X)
X = Flatten()(X)
Features = Dense(3, activation='softmax')(X)
Controller = Lambda(lambda x: K.concatenate([
K.reshape(K.zeros_like(x[:, 0]), (-1, 1)), # to make sure action 0 is dominated
K.reshape(-x[:, 0] - x[:, 1] - 2 * x[:, 2], (-1, 1)),
K.reshape(x[:, 1] + x[:, 2], (-1, 1)),
K.reshape(x[:, 0] + x[:, 2], (-1, 1))]))(Features)
def unet_model_3d(first_input_shape, second_input_shape, nb_classes,
feature_size):
channel_first_first_input = Input(first_input_shape)
first_input = Permute([2, 3, 4, 1])(channel_first_first_input)
first_conv_permute = Permute([4, 2, 3, 1])(first_input)
first_gpooling_0 = GlobalAveragePooling3D()(first_conv_permute)
first_gpooling_dense_0 = Dense(units=32,
activation='linear')(first_gpooling_0)
first_gpooling_dense_1_1 = Dense(
units=29, activation='sigmoid')(first_gpooling_dense_0)
first_gpooling_fused_2 = Lambda(fuse)(
[first_conv_permute, first_gpooling_dense_1_1])
first_conv_layer0 = Conv3D(8, (5, 5, 5),
padding='same',
activation='linear')(first_input)
first_conv_permute = Permute([4, 2, 3, 1])(first_conv_layer0)
first_gpooling_0 = GlobalAveragePooling3D()(first_conv_permute)
first_gpooling_dense_0 = Dense(units=32,
activation='linear')(first_gpooling_0)
first_gpooling_dense_1_0 = Dense(
units=29, activation='sigmoid')(first_gpooling_dense_0)
first_gpooling_fused_0 = Lambda(fuse)(
[first_conv_permute, first_gpooling_dense_1_0])
first_conv_layer1 = Conv3D(8, (3, 3, 3),
padding='same',
activation='linear')(first_conv_layer0)
first_conv_permute = Permute([4, 2, 3, 1])(first_conv_layer1)
first_gpooling_0 = GlobalAveragePooling3D()(first_conv_permute)
first_gpooling_dense_0 = Dense(units=32,
activation='linear')(first_gpooling_0)
first_gpooling_dense_1_1 = Dense(
units=29, activation='sigmoid')(first_gpooling_dense_0)
first_gpooling_fused_1 = Lambda(fuse)(
[first_conv_permute, first_gpooling_dense_1_1])
# 加入自己设计的模块（四）
first_gpooling_add_0 = Add()([
first_gpooling_fused_0, first_gpooling_fused_1, first_gpooling_fused_2
])
first_conv_layer2 = Conv2D(16, (3, 3), padding='same',
activation='linear')(first_gpooling_add_0)
first_pooling_layer1 = MaxPooling2D(pool_size=(2, 2))(first_conv_layer2)
first_conv_layer3 = Conv2D(16, (3, 3), padding='same',
activation='linear')(first_pooling_layer1)
first_pooling_layer2 = MaxPooling2D(pool_size=(2, 2))(first_conv_layer3)
first_conv_layer4 = Conv2D(16, (3, 3), padding='same',
activation='linear')(first_pooling_layer2)
first_pooling_layer3 = MaxPooling2D(pool_size=(2, 2),
padding='same')(first_conv_layer4)
first_flatten_layer1 = Flatten()(first_pooling_layer3)
first_dense_layer1 = Dense(units=feature_size,
activation='relu')(first_flatten_layer1)
first_dense_layer2 = Dense(units=feature_size,
activation='relu')(first_dense_layer1)
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=[128, 128, 3])
second_input = base_model.input
mixed10_output = base_model.output
gpooling = GlobalAveragePooling2D()(mixed10_output)
concat_layer = concatenate([first_dense_layer2, gpooling], axis=1)
input_target = Input(shape=(1, ))
centers = Embedding(nb_classes, feature_size * 3)(input_target)
l2_loss = Lambda(center_loss, name='l2_loss')([concat_layer, centers])
concat_result = Dense(units=nb_classes,
activation='softmax',
name='softmax')(concat_layer)
concat_model = Model(
inputs=[channel_first_first_input, second_input, input_target],
outputs=[concat_result, l2_loss])
concat_test_model = Model(inputs=[channel_first_first_input, second_input],
outputs=concat_result)
# return model_train, model_test, second_train_model
return concat_model, concat_test_model
def get_ESIM_model(nb_words, embedding_dim, embedding_matrix, recurrent_units,
dense_units, dropout_rate, max_sequence_length, out_size):
embedding_layer = Embedding(
nb_words,
embedding_dim,
# embeddings_initializer='uniform',
#weights=[embedding_matrix],
input_length=max_sequence_length,
#trainable=False
)
input_q1_layer = Input(shape=(max_sequence_length, ),
dtype='int32',
name='q1')
input_q2_layer = Input(shape=(max_sequence_length, ),
dtype='int32',
name='q2')
embedding_sequence_q1 = BatchNormalization(axis=2)(
embedding_layer(input_q1_layer))
embedding_sequence_q2 = BatchNormalization(axis=2)(
embedding_layer(input_q2_layer))
final_embedding_sequence_q1 = SpatialDropout1D(0.25)(embedding_sequence_q1)
final_embedding_sequence_q2 = SpatialDropout1D(0.25)(embedding_sequence_q2)
rnn_layer_q1 = Bidirectional(LSTM(
recurrent_units, return_sequences=True))(final_embedding_sequence_q1)
rnn_layer_q2 = Bidirectional(LSTM(
recurrent_units, return_sequences=True))(final_embedding_sequence_q2)
## embedding * embedding
attention = Dot(axes=-1)([rnn_layer_q1, rnn_layer_q2])
print('attention:', attention)
w_attn_1 = Lambda(lambda x: softmax(x, axis=1))(attention) ## 列归一化
w_attn_2 = Permute(
(2, 1))(Lambda(lambda x: softmax(x, axis=2))(attention)) ##行归一化
align_layer_1 = Dot(axes=1)([w_attn_1, rnn_layer_q1])
align_layer_2 = Dot(axes=1)([w_attn_2, rnn_layer_q2])
print('align_layer_1:', align_layer_1)
print('align_layer_1:', align_layer_2)
subtract_layer_1 = subtract([rnn_layer_q1, align_layer_1])
subtract_layer_2 = subtract([rnn_layer_q2, align_layer_2])
multiply_layer_1 = multiply([rnn_layer_q1, align_layer_1])
multiply_layer_2 = multiply([rnn_layer_q2, align_layer_2])
m_q1 = concatenate(
[rnn_layer_q1, align_layer_1, subtract_layer_1, multiply_layer_1])
m_q2 = concatenate(
[rnn_layer_q2, align_layer_2, subtract_layer_2, multiply_layer_2])
v_q1_i = Bidirectional(LSTM(recurrent_units, return_sequences=True))(m_q1)
v_q2_i = Bidirectional(LSTM(recurrent_units, return_sequences=True))(m_q2)
avgpool_q1 = GlobalAveragePooling1D()(v_q1_i)
avgpool_q2 = GlobalAveragePooling1D()(v_q2_i)
maxpool_q1 = GlobalMaxPooling1D()(v_q1_i)
maxpool_q2 = GlobalMaxPooling1D()(v_q2_i)
merged_q1 = concatenate([avgpool_q1, maxpool_q1])
merged_q2 = concatenate([avgpool_q2, maxpool_q2])
final_v = BatchNormalization()(concatenate([merged_q1, merged_q2]))
output = Dense(units=dense_units, activation='relu')(final_v)
output = BatchNormalization()(output)
output = Dropout(dropout_rate)(output)
output = Dense(units=out_size, activation='softmax')(output)
model = Model(inputs=[input_q1_layer, input_q2_layer], output=output)
adam_optimizer = keras.optimizers.Adam(lr=1e-3, decay=1e-6, clipvalue=5)
# parallel_model = multi_gpu_model(model, gpus=2)
# parallel_model.compile(loss='binary_crossentropy', optimizer=adam_optimizer, metrics=['binary_crossentropy', 'accuracy'])
print(model.summary())
# plot(model, 'model.png')
# # model.compile(loss={'output':'binary_crossentropy'}, optimizer=Adam())
# model.compile(loss={'output':'categorical_crossentropy'}, optimizer=Adam(options.lr))
model.compile(loss='categorical_crossentropy', optimizer=adam_optimizer)
return model
def build_model(self,
vocab_size=None,
query_maxlen=None,
story_maxlen=None):
# embed the input sequence into a sequence of vectors
input_encoder_m = Sequential()
input_encoder_m.add(
Embedding(input_dim=vocab_size,
output_dim=64,
input_length=story_maxlen))
input_encoder_m.add(Dropout(0.3))
# output: (samples, story_maxlen, embedding_dim)
# embed the question into a sequence of vectors
question_encoder = Sequential()
question_encoder.add(
Embedding(input_dim=vocab_size,
output_dim=64,
input_length=query_maxlen))
question_encoder.add(Dropout(0.3))
# output: (samples, query_maxlen, embedding_dim)
# compute a 'match' between input sequence elements (which are vectors)
# and the question vector sequence
match = Sequential()
match.add(
Merge([input_encoder_m, question_encoder],
mode='dot',
dot_axes=[2, 2]))
match.add(Activation('softmax'))
# output: (samples, story_maxlen, query_maxlen)
# embed the input into a single vector with size = story_maxlen:
input_encoder_c = Sequential()
input_encoder_c.add(
Embedding(input_dim=vocab_size,
output_dim=query_maxlen,
input_length=story_maxlen))
input_encoder_c.add(Dropout(0.3))
# output: (samples, story_maxlen, query_maxlen)
# sum the match vector with the input vector:
response = Sequential()
response.add(Merge([match, input_encoder_c], mode='sum'))
# output: (samples, story_maxlen, query_maxlen)
response.add(Permute(
(2, 1))) # output: (samples, query_maxlen, story_maxlen)
# concatenate the match vector with the question vector,
# and do logistic regression on top
answer = Sequential()
answer.add(
Merge([response, question_encoder], mode='concat', concat_axis=-1))
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer.add(LSTM(32))
# one regularization layer -- more would probably be needed.
answer.add(Dropout(0.3))
answer.add(Dense(vocab_size))
# we output a probability distribution over the vocabulary
answer.add(Activation('softmax'))
answer.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
return answer
def cnn_rnn(dimx, dimy, num_classes, **kwargs):
"""
Deep Neural Network containing 1 LSTM layers followed by 3 Dense Layers.
Parameters
----------
rnn_units : int
default : 32
Number of Units for LSTM layer.
input_neurons : int
default : 200
Number of Neurons for each Dense layer.
dropout : float
default : 0.1
Dropout used after the Dense Layer.
act1 : str
default : relu
Activation used after Convolution layer.
act2 : str
default : tanh
Activation used after Recurrent layer.
act3 : str
default : softmax
Activation used after Dense layer.
print_sum : bool
default : False
Print summary if the model
nb_filter : int
default : 100
Number of kernels
filter_length : int, tuple
default : 3
Size of kernels
pool_size : int, tuple
default : (2,2)
Pooling size.
loss
default : categorical_crossentropy
Loss used
optimizer
default : adam
Optimizer used
metrics
default : accuracy
Metrics used.
Returns
-------
RNN Model
"""
rnn_units = kwargs['kwargs'].get('rnn_units', 32)
input_neurons = kwargs['kwargs'].get('input_neurons', 200)
act1 = kwargs['kwargs'].get('act1', 'relu')
act2 = kwargs['kwargs'].get('act2', 'tanh')
act3 = kwargs['kwargs'].get('act3', 'softmax')
dropout = kwargs['kwargs'].get('dropout', 0.1)
nb_filter = kwargs['kwargs'].get('nb_filter', 100)
filter_length = kwargs['kwargs'].get('filter_length', 3)
pool_size = kwargs['kwargs'].get('pool_size', (2, 2))
print_sum = kwargs['kwargs'].get('print_sum', False)
loss = kwargs['kwargs'].get('loss', 'categorical_crossentropy')
optimizer = kwargs['kwargs'].get('optimizer', 'adam')
metrics = kwargs['kwargs'].get('metrics', 'accuracy')
main_input = Input(shape=(1, dimx, dimy))
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1)(main_input)
hx = MaxPooling2D(pool_size=pool_size)(x)
wrap = Dropout(dropout)(hx)
x = Permute((2, 1, 3))(wrap)
a, b, c, d = kr(x)
x = Reshape((b * d, c))(x)
x = LSTM(rnn_units, activation=act2)(x)
wrap = Dropout(dropout)(x)
x = Dense(input_neurons, activation=act3)(wrap)
main_output = Dense(num_classes, activation='softmax',
name='main_output')(wrap)
model = Model(inputs=main_input, outputs=main_output)
if print_sum:
model.summary()
model.compile(loss=loss, optimizer=optimizer, metrics=[metrics])
return model
plot(question_net, to_file='question_net.png', show_shapes=True)
merged = Merge([passage_net, question_net], mode='dot')
# merged = Merge([passage_net, question_net], mode='cos')
print("merged layer shape:", question_net.layers[-1].output_shape)
model = Sequential()
model.add(merged)
# multiply passage by the dot product
# add softmax here
# model.add(Dropout(.5))
# model.add(Dense(100, activation='softmax'))
model.add(Permute((2, 1)))
# model.add(MaxPooling1D(pool_length=4, stride=None, border_mode='valid'))
# model.add(Activation('relu'))
model.add(AveragePooling1D(pool_length=40, stride=None, border_mode='valid'))
model.add(Permute((2, 1)))
model.add(Flatten())
# model.add(Highway()) # looks like this kind of worked
# model.add(Dropout(.1))
model.add(Dense(400, activation='softmax'))
plot(model, to_file='model.png', show_shapes=True)
# train a 1D convnet with global maxpooling
# adam = Adam(lr=.0001, clipnorm=10)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
def ACRNN(dimx, dimy, num_classes, **kwargs):
act1 = kwargs['kwargs'].get('act1', 'tanh')
nb_filter = kwargs['kwargs'].get('nb_filter', 72)
filter_length = kwargs['kwargs'].get('filter_length', 4)
act2 = kwargs['kwargs'].get('act2', 'linear')
rnn_units = kwargs['kwargs'].get('rnn_units', [20, 20])
dropout = kwargs['kwargs'].get('dropout', [0.1, 0.2])
act3 = kwargs['kwargs'].get('act3', 'softmax')
print_sum = kwargs['kwargs'].get('print_sum', False)
loss = kwargs['kwargs'].get('loss', 'binary_crossentropy')
optimizer = kwargs['kwargs'].get('optimizer', 'adam')
metrics = kwargs['kwargs'].get('metrics', 'mse')
#input shape
main_input = Input(shape=(1, dimx, dimy))
#CNN
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1,
use_bias=False)(main_input)
hx = MaxPooling2D(pool_size=(1, 2))(x)
x = Conv2D(filters=nb_filter,
kernel_size=filter_length,
data_format='channels_first',
padding='same',
activation=act1,
use_bias=False)(hx)
hx = MaxPooling2D(pool_size=(1, 2))(x)
wrap = Dropout(dropout[0])(hx)
x = Permute((2, 1, 3))(wrap)
a, b, c, d = kr(x)
x = Reshape((b * d, c))(x)
#RNN LAYERS
rnnout = Bidirectional(GRU(rnn_units[0],
activation=act2,
return_sequences=True),
merge_mode='concat')(x)
rnnout_1 = Bidirectional(GRU(rnn_units[1],
activation='sigmoid',
return_sequences=True),
merge_mode='concat')(rnnout)
w = Multiply()([rnnout, rnnout_1])
#Attention starts
hidden_size = int(w._keras_shape[1])
a = Permute((2, 1))(w)
a = Reshape((hidden_size, a._keras_shape[1]))(a)
a = TimeDistributed(
Dense(a._keras_shape[1], activation='softmax', use_bias=False))(a)
a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)
a = RepeatVector(dimy)(a)
a_probs = Permute((2, 1), name='attention_vec')(a)
attention_mul = merge([w, a_probs], name='attention_mul', mode='mul')
attention_mul = GlobalMaxPooling1D()(attention_mul)
attention_mul = Dropout(dropout[1])(attention_mul)
# compile Model
main_output = Dense(num_classes, activation=act3)(attention_mul)
mymodel = Model([main_input], main_output)
if print_sum:
mymodel.summary()
mymodel.compile(loss=loss, optimizer=optimizer, metrics=[metrics])
return mymodel
def encoder_decoder(data):
print('Encoder_Decoder LSTM...')
"""__encoder___"""
encoder_inputs = Input(shape=en_shape)
encoder_LSTM = LSTM(hidden_units,
dropout_U=0.2,
dropout_W=0.2,
return_sequences=True,
return_state=True)
encoder_LSTM_rev = LSTM(hidden_units,
return_state=True,
return_sequences=True,
dropout_U=0.05,
dropout_W=0.05,
go_backwards=True)
encoder_outputs, state_h, state_c = encoder_LSTM(encoder_inputs)
encoder_outputsR, state_hR, state_cR = encoder_LSTM_rev(encoder_inputs)
state_hfinal = Add()([state_h, state_hR])
state_cfinal = Add()([state_c, state_cR])
encoder_outputs_final = Add()([encoder_outputs, encoder_outputsR])
encoder_states = [state_hfinal, state_cfinal]
"""____decoder___"""
decoder_inputs = Input(shape=(None, de_shape[1]))
decoder_LSTM = LSTM(hidden_units,
return_sequences=True,
dropout_U=0.2,
dropout_W=0.2,
return_state=True)
decoder_outputs, _, _ = decoder_LSTM(decoder_inputs,
initial_state=encoder_states)
#Pull out XGBoost, (I mean attention)
attention = TimeDistributed(Dense(
1, activation='tanh'))(encoder_outputs_final)
attention = Flatten()(attention)
attention = Multiply()([decoder_outputs, attention])
attention = Activation('softmax')(attention)
attention = Permute([2, 1])(attention)
decoder_dense = Dense(de_shape[1], activation='softmax')
decoder_outputs = decoder_dense(attention)
model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=decoder_outputs)
print(model.summary())
rmsprop = RMSprop(lr=learning_rate, clipnorm=clip_norm)
model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=['accuracy'])
x_train, x_test, y_train, y_test = tts(data["article"],
data["summaries"],
test_size=0.20)
history = model.fit(x=[x_train, y_train],
y=y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=([x_test, y_test], y_test))
print(model.summary())
"""_________________inference mode__________________"""
encoder_model_inf = Model(encoder_inputs, encoder_states)
decoder_state_input_H = Input(shape=(en_shape[0], ))
decoder_state_input_C = Input(shape=(en_shape[0], ))
decoder_state_inputs = [decoder_state_input_H, decoder_state_input_C]
decoder_outputs, decoder_state_h, decoder_state_c = decoder_LSTM(
decoder_inputs, initial_state=decoder_state_inputs)
decoder_states = [decoder_state_h, decoder_state_c]
decoder_outputs = decoder_dense(decoder_outputs)
decoder_model_inf = Model([decoder_inputs] + decoder_state_inputs,
[decoder_outputs] + decoder_states)
scores = model.evaluate([x_test, y_test], y_test, verbose=1)
print('LSTM test scores:', scores)
print('\007')
print(model.summary())
return model, encoder_model_inf, decoder_model_inf, history
# encode input sequence and questions (which are indices)
# to sequences of dense vectors
input_encoded_m = input_encoder_m(input_sequence)
input_encoded_c = input_encoder_c(input_sequence)
question_encoded = question_encoder(question)
# compute a 'match' between the first input vector sequence
# and the question vector sequence
# shape: `(samples, story_maxlen, query_maxlen)`
match = dot([input_encoded_m, question_encoded], axes=(2, 2))
match = Activation('softmax')(match)
# add the match matrix with the second input vector sequence
response = add([match,
input_encoded_c]) # (samples, story_maxlen, query_maxlen)
response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)
# concatenate the match matrix with the question vector sequence
answer = concatenate([response, question_encoded])
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer = LSTM(32)(answer) # (samples, 32)
# one regularization layer -- more would probably be needed.
answer = Dropout(0.3)(answer)
answer = Dense(vocab_size)(answer) # (samples, vocab_size)
# we output a probability distribution over the vocabulary
answer = Activation('softmax')(answer)
# build the final model
def get_UNET_C_r1(channels, isz, classes, args_dict={}):
# C: customizable UNET, with inception modules
# r1: version
#libraries
from keras.models import Model
from keras.layers import Input, Activation, Dropout, BatchNormalization, concatenate, Reshape, Permute, Lambda
from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, UpSampling2D, Conv2DTranspose
from keras.layers.merge import add
from keras.layers.advanced_activations import ELU, LeakyReLU
from keras.optimizers import Adam
import inspect
# Functions
def NConvolution2D(cc_inputs, nb_conv, nb_filter, args_dict):
# NConvolution2D - Parameters
batch_norm = args_dict.get('batch_norm', False)
bn_pos = args_dict.get('bn_pos', 'before')
conv_activ = args_dict.get('conv_activ', 'all') # all, end
pad = args_dict.get('pad', 0)
kernel = args_dict.get('kernel', 3)
conv_strides = args_dict.get('conv_strides', 1)
conv_padding = args_dict.get('conv_padding', 'same')
init = args_dict.get('init', 'glorot_uniform')
activation = args_dict.get('activation', 'relu')
if pad > 0:
res = ZeroPadding2D(padding=(pad, pad))(cc_inputs)
else:
res = cc_inputs
for i_conv in range(nb_conv):
res = Conv2D(nb_filter, (kernel, kernel),
strides=(conv_strides, conv_strides),
kernel_initializer=init,
padding=conv_padding)(res)
if conv_activ == 'all':
if batch_norm and bn_pos == 'before':
res = BatchNormalization(axis=1)(res)
res = Activation(activation)(res)
if batch_norm and bn_pos == 'after':
res = BatchNormalization(axis=1)(res)
return res
def NConvDilation2D(cc_inputs, nb_conv, nb_filter, args_dict):
# NConvolution2D - Parameters
batch_norm = args_dict.get('batch_norm', False)
bn_pos = args_dict.get('bn_pos', 'before')
pad = args_dict.get('pad', 0)
kernel = args_dict.get('kernel', 3)
conv_strides = args_dict.get('conv_strides', 1)
conv_padding = args_dict.get('conv_padding', 'same')
init = args_dict.get('init', 'glorot_uniform')
activation = args_dict.get('activation', 'relu')
if pad > 0:
res = ZeroPadding2D(padding=(pad, pad))(cc_inputs)
else:
res = cc_inputs
res = Conv2D(nb_filter, (kernel, kernel),
strides=(conv_strides, conv_strides),
kernel_initializer=init,
padding=conv_padding)(res)
for i in range(1, nb_conv):
dr = 2 * i
res = Conv2D(nb_filter, (kernel, kernel),
strides=(conv_strides, conv_strides),
kernel_initializer=init,
padding=conv_padding,
dilation_rate=dr)(res)
if batch_norm and bn_pos == 'before':
res = BatchNormalization(axis=1)(res)
res = Activation(activation)(res)
if batch_norm and bn_pos == 'after':
res = BatchNormalization(axis=1)(res)
return res
def NConcDilation2D(cc_inputs, nb_conv, nb_filter, args_dict):
# NConvolution2D - Parameters
batch_norm = args_dict.get('batch_norm', False)
bn_pos = args_dict.get('bn_pos', 'before')
pad = args_dict.get('pad', 0)
kernel = args_dict.get('kernel', 3)
conv_strides = args_dict.get('conv_strides', 1)
conv_padding = args_dict.get('conv_padding', 'same')
init = args_dict.get('init', 'glorot_uniform')
activation = args_dict.get('activation', 'relu')
if pad > 0:
res = ZeroPadding2D(padding=(pad, pad))(cc_inputs)
else:
res = cc_inputs
res_conc = []
res = Conv2D(nb_filter, (kernel, kernel),
strides=(conv_strides, conv_strides),
kernel_initializer=init,
padding=conv_padding)(res)
res_conc.append(res)
for i in range(1, nb_conv):
dr = 2 * i
res = Conv2D(nb_filter, (kernel, kernel),
strides=(conv_strides, conv_strides),
kernel_initializer=init,
padding=conv_padding,
dilation_rate=dr)(res)
res_conc.append(res)
res = concatenate(res_conc, axis=1)
if batch_norm and bn_pos == 'before':
res = BatchNormalization(axis=1)(res)
res = Activation(activation)(res)
if batch_norm and bn_pos == 'after':
res = BatchNormalization(axis=1)(res)
return res
def inception_block(inputs, depth, splitted=False, activation='relu'):
assert depth % 16 == 0
actv = activation == 'relu' and (lambda: LeakyReLU(
0.0)) or activation == 'elu' and (lambda: ELU(1.0)) or None
c1_1 = Conv2D(depth / 4, (1, 1),
kernel_initializer='he_normal',
padding='same')(inputs)
c2_1 = Conv2D(depth / 8 * 3, (1, 1),
kernel_initializer='he_normal',
padding='same')(inputs)
c2_1 = actv()(c2_1)
if splitted:
c2_2 = Conv2D(depth / 2, (1, 3),
kernel_initializer='he_normal',
padding='same')(c2_1)
c2_2 = BatchNormalization(axis=1)(c2_2)
c2_2 = actv()(c2_2)
c2_3 = Conv2D(depth / 2, (3, 1),
kernel_initializer='he_normal',
padding='same')(c2_2)
else:
c2_3 = Conv2D(depth / 2, (3, 3),
kernel_initializer='he_normal',
padding='same')(c2_1)
c3_1 = Conv2D(depth / 16, (1, 1),
kernel_initializer='he_normal',
padding='same')(inputs)
#missed batch norm
c3_1 = actv()(c3_1)
if splitted:
c3_2 = Conv2D(depth / 8, (1, 5),
kernel_initializer='he_normal',
padding='same')(c3_1)
c3_2 = BatchNormalization(axis=1)(c3_2)
c3_2 = actv()(c3_2)
c3_3 = Conv2D(depth / 8, (5, 1),
kernel_initializer='he_normal',
padding='same')(c3_2)
else:
c3_3 = Conv2D(depth / 8, (5, 5),
kernel_initializer='he_normal',
padding='same')(c3_1)
p4_1 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1),
padding='same')(inputs)
c4_2 = Conv2D(depth / 8, (1, 1),
kernel_initializer='he_normal',
padding='same')(p4_1)
res = concatenate([c1_1, c2_3, c3_3, c4_2], axis=1)
res = BatchNormalization(axis=1)(res)
res = actv()(res)
return res
def _shortcut(_input, residual):
stride_width = _input._keras_shape[2] / residual._keras_shape[2]
stride_height = _input._keras_shape[3] / residual._keras_shape[3]
equal_channels = residual._keras_shape[1] == _input._keras_shape[1]
shortcut = _input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv2D(residual._keras_shape[1], (1, 1),
strides=(stride_width, stride_height),
kernel_initializer="he_normal",
padding="valid")(_input)
return add([shortcut, residual])
def rblock(inputs, num, depth, scale=0.1):
residual = Conv2D(depth, (num, num), padding='same')(inputs)
residual = BatchNormalization(axis=1)(residual)
residual = Lambda(lambda x: x * scale,
output_shape=lambda x: x)(residual)
res = _shortcut(inputs, residual)
return ELU()(res)
# Unet - Parameters
convs = args_dict.get('convs',
[[2, 2, 2, 2], 1, [2, 2, 2, 2]]) #down-mid-up
filters = args_dict.get(
'filters', [[16, 32, 64, 128], 256, [128, 64, 32, 16]]) #down-mid-up
drops = args_dict.get('drops',
[[0, 0, 0, 0], 0, [0, 0, 0, 0]]) #down-mid-up
down_size = args_dict.get('down_size', 2)
up_size = args_dict.get('up_size', 2)
up_drop_pos = args_dict.get('up_drop_pos',
'after_conv') #after_conv, before_conv
learnable = args_dict.get('learnable', False)
strides = args_dict.get('strides', False)
strides_conc = args_dict.get('strides_conc', 'conv') # conv, pool, input,
strides_type = args_dict.get('strides_type', '') # '', residual, vgg
strides_vgg_convs = args_dict.get('strides_vgg_convs', [1, 1, 1, 1])
strides_vgg_filters = args_dict.get('strides_vgg_filters', [8, 16, 32, 64])
type_filters = args_dict.get('type_filters',
'') #'', ConvDila, ConcDila, inception
conv_activ = args_dict.get('conv_activ', 'all') # all, end
init = args_dict.get('init', 'glorot_uniform')
activation = args_dict.get('activation', 'relu')
batch_norm = args_dict.get('batch_norm', False)
bn_pos = args_dict.get('bn_pos', 'before')
final_activation = args_dict.get('final_activation', 'sigmoid')
flat_output = args_dict.get('flat_output', False)
compile_conf = args_dict.get('compile_conf', 0)
optimizer = args_dict.get('optimizer', Adam())
optimizer = optimizer() if inspect.isclass(optimizer) else optimizer
loss = args_dict.get('loss', 'binary_crossentropy')
metrics = args_dict.get('metrics', [])
inputs = Input((channels, isz[0], isz[1]))
# Down sampling
net = inputs
down_inputs = []
down_convs = []
down_pools = []
down_pools.append(None)
for i in range(len(filters[0])):
down_inputs.append(net)
if type_filters == 'ConvDila':
conv = NConvDilation2D(net, convs[0][i], filters[0][i], args_dict)
down_convs.append(conv)
if type_filters == 'ConcDila':
conv = NConcDilation2D(net, convs[0][i], filters[0][i], args_dict)
down_convs.append(conv)
if type_filters == 'inception':
conv = inception_block(net,
filters[0][i],
splitted=True,
activation=activation)
down_convs.append(conv)
else:
conv = NConvolution2D(net, convs[0][i], filters[0][i], args_dict)
down_convs.append(conv)
if conv_activ == 'end':
if batch_norm and bn_pos == 'before':
conv = BatchNormalization(axis=1)(conv)
conv = Activation(activation)(conv)
if batch_norm and bn_pos == 'after':
conv = BatchNormalization(axis=1)(conv)
if learnable:
pool = Conv2D(filters[0][i], (3, 3),
strides=(down_size, down_size),
kernel_initializer=init,
padding='same')(conv)
else:
pool = MaxPooling2D(pool_size=(down_size, down_size))(conv)
down_pools.append(pool)
net = pool
if drops[0][i] > 0:
net = Dropout(drops[0][i])(net)
# Mid
if type_filters == 'ConvDila':
conv = NConvDilation2D(net, convs[1], filters[1], args_dict)
if type_filters == 'ConcDila':
conv = NConcDilation2D(net, convs[1], filters[1], args_dict)
if type_filters == 'inception':
conv = inception_block(net,
filters[1],
splitted=True,
activation=activation)
else:
conv = NConvolution2D(net, convs[1], filters[1], args_dict)
if conv_activ == 'end':
if batch_norm and bn_pos == 'before':
conv = BatchNormalization(axis=1)(conv)
conv = Activation(activation)(conv)
if batch_norm and bn_pos == 'after':
conv = BatchNormalization(axis=1)(conv)
net = conv
if drops[1] > 0:
net = Dropout(drops[1])(net)
# Up sampling
inv = list(reversed(range(len(filters[0]))))
up_ch = filters[1]
for i in range(len(filters[2])):
if learnable:
up = Conv2DTranspose(up_ch, (3, 3),
strides=(2, 2),
kernel_initializer=init,
padding="same")(net)
else:
up = UpSampling2D(size=(up_size, up_size))(net)
if strides:
if strides_conc == 'conv':
net_conc = down_convs[inv[i]]
elif strides_conc == 'pool':
net_conc = down_pools[inv[i]]
elif strides_conc == 'input':
net_conc = down_inputs[inv[i]]
if strides_type == 'residual' and net_conc is not None:
net_conc = rblock(net_conc, 1, filters[2][i])
elif strides_type == 'vgg' and net_conc is not None:
net_conc = NConvDilation2D(net_conc, strides_vgg_convs[inv[i]],
strides_vgg_filters[inv[i]],
args_dict)
if net_conc is None:
conc = up
else:
conc = concatenate([up, net_conc], axis=1)
else:
conc = up
if up_drop_pos == 'before_conv' and drops[2][i] > 0:
conc = Dropout(drops[2][i])(conc)
if type_filters == 'ConvDila':
conv = NConvDilation2D(conc, convs[2][i], filters[2][i], args_dict)
if type_filters == 'ConcDila':
conv = NConcDilation2D(conc, convs[2][i], filters[2][i], args_dict)
if type_filters == 'inception':
conv = inception_block(conc,
filters[2][i],
splitted=True,
activation=activation)
else:
conv = NConvolution2D(conc, convs[2][i], filters[2][i], args_dict)
if conv_activ == 'end':
if batch_norm and bn_pos == 'before':
conv = BatchNormalization(axis=1)(conv)
conv = Activation(activation)(conv)
if batch_norm and bn_pos == 'after':
conv = BatchNormalization(axis=1)(conv)
up_ch = filters[2][i]
net = conv
if up_drop_pos == 'after_conv' and drops[2][i] > 0:
net = Dropout(drops[2][i])(net)
if flat_output:
final = Conv2D(classes, (1, 1),
strides=(1, 1),
kernel_initializer=init,
padding="same")(net)
final = Reshape((classes, isz[0] * isz[1]))(final)
final = Permute((2, 1))(final)
final = Activation(final_activation)(final)
else:
final = Conv2D(classes, (1, 1),
strides=(1, 1),
kernel_initializer=init,
padding="same")(net)
final = Activation(final_activation)(final)
model = Model(inputs=inputs, outputs=final)
if compile_conf == 0:
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
if compile_conf == 1:
model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics,
sample_weight_mode="temporal")
return model
def create_layer(self):
if self.type == "Dense":
self.block = Dense(units=int(self.gff("units")), activation=self.gff("activation"))
elif self.type == "Input":
self.block = Input(shape=self.get_int(self.gff("shape")))
elif self.type == "Dropout":
self.block = Dropout(rate=float(self.gff("rate")))
elif self.type == "Flatten":
self.block = Flatten()
elif self.type == "Reshape":
self.block = Reshape(target_shape=int(self.gff("rate")))
elif self.type == "Permute":
self.block = Permute(dims=self.get_int(self.gff("dims")))
elif self.type == "RepeatVector":
self.block = RepeatVector(n=int(self.gff("dims")))
#CONVOLUTIONAL
elif self.type == "Conv1D":
self.block = Conv1D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "Conv2D":
self.block = Conv2D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "Conv2DTranspose":
self.block = Conv2DTranspose(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "Conv3D":
self.block = Conv3D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "UpSampling1D":
self.block = UpSampling1D(size=self.get_int(self.gff("size")))
elif self.type == "UpSampling2D":
self.block = UpSampling2D(size=self.get_int(self.gff("size")))
elif self.type == "UpSampling3D":
self.block = UpSampling3D(size=self.get_int(self.gff("size")))
#POOLING
elif self.type == "MaxPooling1D":
self.block = MaxPooling1D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "MaxPooling2D":
self.block = MaxPooling2D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "MaxPooling3D":
self.block = MaxPooling3D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "AveragePooling1D":
self.block = AveragePooling1D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "AveragePooling2D":
self.block = AveragePooling2D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "AveragePooling3D":
self.block = AveragePooling3D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "GlobalMaxPooling1D":
self.block = GlobalMaxPooling1D()
elif self.type == "GlobalMaxPooling2D":
self.block = GlobalMaxPooling2D()
elif self.type == "GlobalAveragePooling1D":
self.block = GlobalAveragePooling1D()
elif self.type == "GlobalAveragePooling2D":
self.block = GlobalAveragePooling2D()
#Locally Connected
elif self.type == "LocallyConnected1D":
self.block = LocallyConnected1D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "LocallyConnected2D":
self.block = LocallyConnected2D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
#MERGE LAYERS
elif self.type == "Add":
self.block = Add()
elif self.type == "Subtract":
self.block = Subtract()
elif self.type == "Multiply":
self.block = Multiply()
elif self.type == "Average":
self.block = Average()
elif self.type == "Maximum":
self.block = Maximum()
elif self.type == "Concatenate":
self.block = Concatenate()
elif self.type == "Dot":
self.block = Dot()
#NORMALISATION LAYER
elif self.type == "BatchNormalization":
self.block = BatchNormalization(axis=int(self.gff("axis")), center=bool(self.gff("center")),
momentum=float(self.gff("momentum")), epsilon=float(self.gff("epsilon")),
scale=bool(self.gff("scale")))
#NOISE LAYERS
elif self.type == "GaussianNoise":
self.block = GaussianNoise(stddev=self.get_float(self.gff("stddev")))
#NOISE LAYERS
elif self.type == "GaussianDropout":
self.block = GaussianDropout(rate=self.get_float(self.gff("rate")))
elif self.type == "AlphaDropout":
self.block = AlphaDropout(rate=self.get_float(self.gff("rate")), seed=int(self.gff("rate")))
def branch_cnn_am1(self, q1, q2, X_train_q1, X_train_q2):
emb_layer = Embedding(self.MAX_TEXT, self.emb_size, trainable=True)
emb_q1 = emb_layer(q1)
emb_q2 = emb_layer(q2)
match_score = self.MatchScore(emb_q1, emb_q2, mode='cos')
attention_left = TimeDistributed(
Dense(self.emb_size, activation="tanh"),
input_shape=(X_train_q1.shape[1],
X_train_q2.shape[1]))(match_score)
match_score_t = Permute((2, 1))(match_score)
attention_right = TimeDistributed(
Dense(self.emb_size, activation="tanh"),
input_shape=(X_train_q2.shape[1],
X_train_q1.shape[1]))(match_score_t)
left_reshape = Reshape((1, attention_left._keras_shape[1],
attention_left._keras_shape[2]))
attention_left = left_reshape(attention_left)
emb_q1 = left_reshape(emb_q1)
right_reshape = Reshape((1, attention_right._keras_shape[1],
attention_right._keras_shape[2]))
attention_right = right_reshape(attention_right)
emb_q2 = right_reshape(emb_q2)
emb_q1 = merge([emb_q1, attention_left], mode="concat", concat_axis=1)
emb_q2 = merge([emb_q2, attention_right], mode="concat", concat_axis=1)
left_embed_padded = ZeroPadding2D((int(3 / 2), 0))(emb_q1)
right_embed_padded = ZeroPadding2D((int(3 / 2), 0))(emb_q2)
conv_left = Conv2D(filters=64,
kernel_size=(3, self.emb_size),
activation="tanh",
padding="valid")(left_embed_padded)
conv_left = (Reshape(
(conv_left._keras_shape[1], conv_left._keras_shape[2])))(conv_left)
conv_left = AveragePooling1D(pool_size=3, strides=1,
padding='same')(conv_left)
# text 1d convolution
conv_left = Conv1D(128, 3, strides=1, padding='valid')(conv_left)
conv_left = Activation('relu')(conv_left)
conv_left = MaxPooling1D(pool_size=2)(conv_left)
conv_left = Dropout(0.2)(conv_left)
conv_left = Conv1D(32, 3, strides=1, padding='valid')(conv_left)
conv_left = Activation('relu')(conv_left)
conv_left = MaxPooling1D(pool_size=2)(conv_left)
# conv_right
conv_right = Conv2D(filters=64,
kernel_size=(3, self.emb_size),
activation="tanh",
padding="valid")(right_embed_padded)
conv_right = (Reshape((conv_right._keras_shape[1],
conv_right._keras_shape[2])))(conv_right)
conv_right = AveragePooling1D(pool_size=3, strides=1,
padding='same')(conv_right)
conv_right = Conv1D(128,
3,
strides=1,
padding='valid',
activation='relu')(conv_right)
conv_right = MaxPooling1D(pool_size=2)(conv_right)
conv_right = Dropout(0.2)(conv_right)
conv_right = Conv1D(32,
3,
strides=1,
padding='valid',
activation='relu')(conv_right)
conv_right = MaxPooling1D(pool_size=2)(conv_right)
cnn = concatenate([conv_left, conv_right])
return cnn
def block_warp(block_name,input_layer,filters,kernal_size=3, dilation_rate=1,depthfilter=4,stride=1):
def conv_block(input_layer,filters,k=3):
y = Conv3D(filters=filters, kernel_size=k, padding='same')(input_layer)
y = BatchNormalization()(y)
y = Activation('relu')(y)
return y
if block_name == 'conv':
y = conv_block(input_layer,filters)
y = conv_block(y,filters)
elif block_name == 'dialtion':
y = Conv3D(filters=filters, kernel_size=kernal_size, padding='same', dilation_rate=dilation_rate)(input_layer)
y = BatchNormalization()(y)
y = Activation('relu')(y)
elif block_name == 'deconv':
y = Conv3DTranspose(filters=filters,kernel_size=3,strides=2,padding='same')(input_layer)
y = BatchNormalization()(y)
y = Activation('relu')(y)
elif block_name == 'time_conv':
y = TimeDistributed(Conv2D(filters=filters,kernel_size=3,padding='same'))(input_layer)
y = TimeDistributed(BatchNormalization())(y)
y = TimeDistributed(Activation('relu'))(y)
elif block_name == 'time_deconv':
y = TimeDistributed(Conv2DTranspose(filters=filters,kernel_size=3,padding='same',strides=2))(input_layer)
y = TimeDistributed(BatchNormalization())(y)
y = TimeDistributed(Activation('relu'))(y)
elif block_name == 'inception':
filters = filters//4
c1 = conv_block(input_layer,filters,1)
c3 = conv_block(input_layer,filters,1)
c3 = conv_block(c3,filters,3)
c5 = MaxPool3D(pool_size=3,padding='same',strides=1)(input_layer)
c5 = conv_block(c5,filters,3)
c7 = conv_block(input_layer,filters,1)
c7 = conv_block(c7, filters, 3)
c7 = conv_block(c7, filters, 3)
y = concatenate([c1,c3,c5,c7])
# c_all = BatchNormalization()(c_all)
# y = Activation('relu')(c_all)
elif block_name == 'sep':
input_layer = Permute((4,1,2,3))(input_layer)
sz = input_layer.get_shape()
shape = tuple(int(sz[i]) for i in range(1, 5))
input_layer = Reshape(shape+(1,))(input_layer)
conv1 = TimeDistributed(Conv3D(filters=depthfilter,kernel_size=kernal_size,padding='same',strides=stride))(input_layer)
conv1 = Permute((2,3,4,1,5))(conv1)
sz = conv1.get_shape()
shape = tuple(int(sz[i]) for i in range(1, 4))
conv1 = Reshape(shape+(-1,))(conv1)
conv1 = Conv3D(filters=filters,kernel_size=1,padding='same')(conv1)
conv1 = BatchNormalization()(conv1)
y = Activation('relu')(conv1)
else:
raise ValueError("layer error")
return y
def getTimitModel2D(d):
n = d.num_layers
sf = d.start_filter
activation = d.act
advanced_act = d.aact
drop_prob = d.dropout
inputShape = (3, 41, None)
filsize = (3, 5)
channelAxis = 1
if d.aact != "none":
d.act = 'linear'
convArgs = {
"activation": d.act,
"data_format": "channels_first",
"padding": "same",
"bias_initializer": "zeros",
"kernel_regularizer": l2(d.l2),
"kernel_initializer": "random_uniform",
}
denseArgs = {
"activation": d.act,
"kernel_regularizer": l2(d.l2),
"kernel_initializer": "random_uniform",
"bias_initializer": "zeros",
"use_bias": True
}
if d.model == "quaternion":
convArgs.update({"kernel_initializer": d.quat_init})
#
# Input Layer & CTC Parameters for TIMIT
#
if d.model == "quaternion":
I = Input(shape=(4, 41, None))
else:
I = Input(shape=inputShape)
labels = Input(name='the_labels', shape=[None], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
#
# Input stage:
#
if d.model == "real":
O = Conv2D(sf, filsize, name='conv', use_bias=True, **convArgs)(I)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
else:
O = QuaternionConv2D(sf,
filsize,
name='conv',
use_bias=True,
**convArgs)(I)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
#
# Pooling
#
O = keras.layers.MaxPooling2D(pool_size=(1, 3), padding='same')(O)
#
# Stage 1
#
for i in xrange(0, n / 2):
if d.model == "real":
O = Conv2D(sf,
filsize,
name='conv' + str(i),
use_bias=True,
**convArgs)(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
else:
O = QuaternionConv2D(sf,
filsize,
name='conv' + str(i),
use_bias=True,
**convArgs)(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
#
# Stage 2
#
for i in xrange(0, n / 2):
if d.model == "real":
O = Conv2D(sf * 2,
filsize,
name='conv' + str(i + n / 2),
use_bias=True,
**convArgs)(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
else:
O = QuaternionConv2D(sf * 2,
filsize,
name='conv' + str(i + n / 2),
use_bias=True,
**convArgs)(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
#
# Permutation for CTC
#
O = Permute((3, 1, 2))(O)
O = Lambda(lambda x: K.reshape(x, (K.shape(x)[0], K.shape(x)[1], K.shape(x)
[2] * K.shape(x)[3])),
output_shape=lambda x: (None, None, x[2] * x[3]))(O)
#
# Dense
#
if d.model == "quaternion":
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
else:
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(d.dropout)(O)
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if d.aact == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
pred = TimeDistributed(
Dense(62,
activation='softmax',
kernel_regularizer=l2(d.l2),
use_bias=True,
bias_initializer="zeros",
kernel_initializer='random_uniform'))(O)
#
# CTC For sequence labelling
#
O = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([pred, labels, input_length, label_length])
# Return the model
#
# Creating a function for testing and validation purpose
#
val_function = K.function([I], [pred])
return Model(inputs=[I, input_length, labels, label_length],
outputs=O), val_function
def get_resnet_model(save_path, model_res=1024, image_size=256, depth=2, size=0, activation='elu', loss='logcosh', optimizer='adam'):
# Build model
if os.path.exists(save_path):
print('Loading model')
model = load_model(save_path)
model.compile(loss=loss, metrics=[], optimizer=optimizer) # By default: adam optimizer, logcosh used for loss.
return model
print('Building model')
model_scale = int(2*(math.log(model_res,2)-1)) # For example, 1024 -> 18
if size <= 0:
from keras.applications.resnet50 import ResNet50
resnet = ResNet50(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3))
else:
from keras_applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
if size == 1:
resnet = ResNet50V2(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3), backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils)
if size == 2:
resnet = ResNet101V2(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3), backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils)
if size >= 3:
resnet = ResNet152V2(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3), backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils)
layer_size = model_scale*8*8*8
if is_square(layer_size): # work out layer dimensions
layer_l = int(math.sqrt(layer_size)+0.5)
layer_r = layer_l
else:
layer_m = math.log(math.sqrt(layer_size),2)
layer_l = 2**math.ceil(layer_m)
layer_r = layer_size // layer_l
layer_l = int(layer_l)
layer_r = int(layer_r)
x_init = None
inp = Input(shape=(image_size, image_size, 3))
x = resnet(inp)
if (depth < 0):
depth = 1
if (size <= 1):
if (size <= 0):
x = Conv2D(model_scale*8, 1, activation=activation)(x) # scale down
x = Reshape((layer_r, layer_l))(x)
else:
x = Conv2D(model_scale*8*4, 1, activation=activation)(x) # scale down a little
x = Reshape((layer_r*2, layer_l*2))(x)
else:
if (size == 2):
x = Conv2D(1024, 1, activation=activation)(x) # scale down a bit
x = Reshape((256, 256))(x)
else:
x = Reshape((256, 512))(x) # all weights used
while (depth > 0): # See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py - TreeConnect inspired layers instead of dense layers.
x = LocallyConnected1D(layer_r, 1, activation=activation)(x)
x = Permute((2, 1))(x)
x = LocallyConnected1D(layer_l, 1, activation=activation)(x)
x = Permute((2, 1))(x)
if x_init is not None:
x = Add()([x, x_init]) # add skip connection
x_init = x
depth-=1
x = Reshape((model_scale, 512))(x) # train against all dlatent values
model = Model(inputs=inp,outputs=x)
model.compile(loss=loss, metrics=[], optimizer=optimizer) # By default: adam optimizer, logcosh used for loss.
return model
def buildFCN(model,imgSize=224,categories=21):
#os
model.add(Permute((1,2,3),input_shape = (imgSize,imgSize,3)))
# Downsampling path #
#1st block
#Adding convolution layers
model.add(Convolution2D(64,kernel_size = (3,3),padding = "same",activation = "relu",name = "block1_conv1"))
model.add(Convolution2D(64,kernel_size = (3,3),padding = "same",activation = "relu",name = "block1_conv2"))
#Addding max pooling layer
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),name = 'block1_pool'))
#2nd block
#Adding convolution layers
model.add(Convolution2D(128,kernel_size = (3,3),padding = "same",activation = "relu",name = "block2_conv1"))
model.add(Convolution2D(128,kernel_size = (3,3),padding = "same",activation = "relu",name = "block2_conv2"))
#Addding max pooling layer
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),name = 'block2_pool'))
#3rd block
#Adding convolution layers
model.add(Convolution2D(256,kernel_size = (3,3),padding = "same",activation = "relu",name = "block3_conv1"))
model.add(Convolution2D(256,kernel_size = (3,3),padding = "same",activation = "relu",name = "block3_conv2"))
model.add(Convolution2D(256,kernel_size = (3,3),padding = "same",activation = "relu",name = "block3_conv3"))
#Addding max pooling layer
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),name = 'block3_pool'))
#4th block
#Adding convolution layers
model.add(Convolution2D(512,kernel_size = (3,3),padding = "same",activation = "relu",name = "block4_conv1"))
model.add(Convolution2D(512,kernel_size = (3,3),padding = "same",activation = "relu",name = "block4_conv2"))
model.add(Convolution2D(512,kernel_size = (3,3),padding = "same",activation = "relu",name = "block4_conv3"))
#Addding max pooling layer
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),name = 'block4_pool'))
#5th block
#Adding convolution layers
model.add(Convolution2D(512,kernel_size = (3,3),padding = "same",activation = "relu",name = "block5_conv1"))
model.add(Convolution2D(512,kernel_size = (3,3),padding = "same",activation = "relu",name = "block5_conv2"))
model.add(Convolution2D(512,kernel_size = (3,3),padding = "same",activation = "relu",name = "block5_conv3"))
#Adding max pooling layer
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),name = 'block5_pool'))
model.add(Convolution2D(4096,kernel_size=(7,7),padding = "same",activation = "relu",name = "fc6"))
#Replacing fully connnected layers of VGG Net using convolutions
model.add(Convolution2D(4096,kernel_size=(1,1),padding = "same",activation = "relu",name = "fc7"))
# Gives the classifications scores for each of the N classes including background
model.add(Convolution2D(categories,kernel_size=(1,1),padding="same",activation="relu",name = "score_fr"))
#Save convolution size
desiredSize=model.layers[-1].output_shape[2]
# Upsampling #
#First deconv layer
model.add(Deconvolution2D(categories,kernel_size=(4,4),strides = (2,2),padding = "valid"))
actualSize=model.layers[-1].output_shape[2]
extra=actualSize-2*desiredSize
#Cropping to get correct size
model.add(Cropping2D(cropping=((0,extra),(0,extra))))
Conv_size = model.layers[-1].output_shape[2]
#Conv to be applied on Pool4
skip_con1 = Convolution2D(categories,kernel_size=(1,1),padding = "same",activation=None, name = "score_pool4")
#Addig skip connection which takes adds the output of Max pooling layer 4 to current layer
Summed = add(inputs = [skip_con1(model.layers[14].output),model.layers[-1].output])
#Upsampling output of first skip connection
x = Deconvolution2D(categories,kernel_size=(4,4),strides = (2,2),padding = "valid",activation=None,name = "score4")(Summed)
x = Cropping2D(cropping=((0,2),(0,2)))(x)
#Conv to be applied to pool3
skip_con2 = Convolution2D(categories,kernel_size=(1,1),padding = "same",activation=None, name = "score_pool3")
#Adding skip connection which takes output og Max pooling layer 3 to current layer
Summed = add(inputs = [skip_con2(model.layers[10].output),x])
#Final Up convolution which restores the original image size
Up = Deconvolution2D(categories,kernel_size=(16,16),strides = (8,8),
padding = "valid",activation = None,name = "upsample")(Summed)
#Cropping the extra part obtained due to transpose convolution
final = Cropping2D(cropping = ((0,8),(0,8)))(Up)
return Model(model.input, final)
def image_entry_model_37(time_steps, data_dim):
inputs = Input(shape=(time_steps, data_dim, 3))
x_0 = inputs
def Conv_filters(x_0):
x_1_1 = Conv2D(filters=16, kernel_size=(3, 4), strides=(2, 2), padding="same", activation="relu")(x_0)
x_1_2 = Conv2D(filters=16, kernel_size=(3, 2), strides=(2, 2), padding="same", activation="relu")(x_0)
x_1 = average([x_1_1, x_1_2])
x_2_1 = Conv2D(filters=16, kernel_size=(3, 4), strides=(2, 2), padding="same", activation="relu")(x_1)
x_2_2 = Conv2D(filters=16, kernel_size=(3, 2), strides=(2, 2), padding="same", activation="relu")(x_1)
x_2 = average([x_2_1, x_2_2])
x_3_1 = Conv2D(filters=16, kernel_size=(3, 4), strides=(2, 2), padding="same", activation="relu")(x_2)
x_3_2 = Conv2D(filters=16, kernel_size=(3, 2), strides=(2, 2), padding="same", activation="relu")(x_2)
x_3 = average([x_3_1, x_3_2])
x_4 = MaxPooling2D(pool_size=(3, 3), padding="same")(x_3)
x_4 = Dropout(0.5)(x_4)
return x_4
x_4_1 = Conv_filters(x_0)
x_4_1 = Permute((3, 2, 1))(x_4_1)
x_shape = K.int_shape(x_4_1)
x_4_1 = [Lambda(slicer_3D_0.slice_pieces_3D, output_shape=(x_shape[1],x_shape[2], 1))(x_4_1) for _ in range(x_shape[3])]
x_4_1 = [Reshape((1, K.int_shape(Flatten()(each))[1]))(Flatten()(each)) for each in x_4_1]
x_4_1 = concatenate(x_4_1, axis=1)
x_5_1 = GRU(256, return_sequences=True)(x_4_1)
x_6_1 = GRU(128)(x_5_1)
x_4_2 = Conv_filters(x_0)
x_shape = K.int_shape(x_4_2)
x_4_2 = [Lambda(slicer_3D_1.slice_pieces_3D, output_shape=(x_shape[1],x_shape[2], 1))(x_4_2) for _ in range(x_shape[3])]
x_4_2 = [Reshape((1, K.int_shape(Flatten()(each))[1]))(Flatten()(each)) for each in x_4_2]
x_4_2 = concatenate(x_4_2, axis=1)
x_5_2 = GRU(256, return_sequences=True)(x_4_2)
x_6_2 = GRU(128)(x_5_2)
x_6 = concatenate([x_6_1, x_6_2])
x_7 = Dense(128, activation="relu")(x_6)
prediction = Dense(4, activation="softmax")(x_7)
model = Model(inputs=inputs, outputs=prediction)
model.summary()
'''
train = snore_data_extractor(load_folder_path, one_hot=True, data_mode="train", resize=(data_dim, time_steps), timechain=False, duplicate=True)
devel = snore_data_extractor(load_folder_path, one_hot=True, data_mode="devel", resize=(data_dim, time_steps), timechain=False, duplicate=True)
epoch_num = 500
batch_size = 16
loss = tf.reduce_mean(losses.kullback_leibler_divergence(labels, predicts))
train_step = tf.train.RMSPropOptimizer(learning_rate=0.001, momentum=0.01).minimize(loss)
'''
return model
pooling_2 = MaxPooling1D(pool_size=4)(trigram_branch)
fourgram_branch = Conv1D(filters=100,
kernel_size=4,
padding='same',
activation='relu',
strides=1)(embedding)
pooling_3 = MaxPooling1D(pool_size=4)(fourgram_branch)
merged = concatenate([pooling_1, pooling_2, pooling_3], axis=1)
activations = (Bidirectional(
LSTM(100, return_sequences=True,
input_shape=(max_length, 100))))(merged)
attention = Dense(1, activation='tanh')(activations)
attention = Flatten()(attention)
attention = Activation('softmax')(attention)
attention = RepeatVector(200)(attention)
attention = Permute([2, 1])(attention)
attn_Multiply = multiply([activations, attention])
sent_representation = Lambda(lambda xin: K.sum(xin, axis=1))(
attn_Multiply)
probabilities = Dense(2, activation='softmax')(sent_representation)
model = Model(inputs=input_1, outputs=probabilities)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print "The model summary is ", model.summary()
filepath = 'CNN_BiSTM_Attn' + '_' + str(epoch) + '_' + str(
bs) + ".hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
Y_valid = np.zeros(
(data_info['num_valid_samples'], data_info['predict_time']))
for i, ind in enumerate(data_info['train_idx']):
X_train[i, :, :] = X[ind, :, :]
Y_train[i, :] = Y[ind, :]
for i, ind in enumerate(data_info['valid_idx']):
X_valid[i, :, :] = X[ind, :, :]
Y_valid[i, :] = Y[ind, :]
inp = Input(shape=(
data_info['train_time'],
data_info['num_features'],
))
trans = Permute((2, 1))(inp)
GRU1 = GRU(20, return_sequences=True)(trans)
drop1 = Dropout(rate=.3)(GRU1)
conv1 = Conv1D(20, 3, padding="valid")(drop1)
LSTM1 = LSTM(20, return_sequences=True)(conv1)
flat1 = Flatten()(LSTM1)
dense1 = Dense(20, activation="linear")(flat1)
out = Dense(data_info['predict_time'], activation="linear")(dense1)
earlystop = EarlyStopping(monitor='val_loss',
min_delta=.01,
patience=2,
verbose=1,
from keras import backend as K import tensorflow as tf import os import numpy as np input_dim = (32, 32, 3) img_inputs = Input(shape=input_dim) conv1 = Conv2D(1, (1, 1), padding='same', activation='relu')(img_inputs) #1 conv64 = Conv2D(9, (3, 3), padding='same', activation='relu')(conv1) #2 """ This found the mistake of softmax returning array on 1s. This solution will basically find the row importance and then the features' importance. """ y = Conv2D(1, (1, 1))(conv64) # 32x32x1 ? y = Permute((3, 2, 1))(y) y = Dense(32, activation='softmax')(y) y = Permute((1, 3, 2))(y) y = Dense(32, activation='softmax')(y) #now permute back y = Permute((1, 3, 2))(y) y = Permute((3, 2, 1))(y) # mult = Multiply()([conv64,y]) # # summed = Lambda(lambda x: K.sum(x, axis=(1,2)), output_shape=lambda s: (s[0], s[3]))(mult) # # dense5 = Dense(64, activation='relu')(summed) # final = Dense(10, activation='softmax')(dense5)
