def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True):
conv_a = Conv2D(
n_filts,
(self.config.filt_time_width, self.config.filt_freq_width),
strides=(
self.config.filt_time_stride,
self.config.filt_freq_stride,
),
padding='same',
name=name_prefix + '_a',
**_layer_kwargs()
)(last_layer)
conv_b = Conv2D(
n_filts,
(self.config.filt_time_width, self.config.filt_freq_width),
strides=(
self.config.filt_time_stride,
self.config.filt_freq_stride,
),
padding='same',
name=name_prefix + '_b',
**_layer_kwargs()
)(last_layer)
last = Maximum(name=name_prefix + '_m')([conv_a, conv_b])
# pre-weights (i.e. post max), as per
# http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf
if dropout:
last = Dropout(
train_config.dropout, name=name_prefix + '_d',
seed=self._cur_weight_seed)(last)
self._cur_weight_seed += 1
return last
def build_generator(self):
example = Input(shape=(self.feature_dims, ))
noise = Input(shape=(self.z_dims, ))
x = Concatenate(axis=1)([example, noise])
for dim in self.generator_layers[1:]:
x = Dense(dim)(x)
x = Activation(activation='sigmoid')(x)
x = Maximum()([example, x])
generator = Model([example, noise], x, name='generator')
generator.summary()
return generator
def build_generator(self, n_hidden=256, h_activation='relu',
regularizers={}, batchnorm=False,
out_activation='sigmoid',
drop_rate=0, summary=False):
"""Builds a generator using the passed hyperparameters"""
# Input: xz, z, or x
x = Input(shape=(self.x_dim,), name='g_x_input')
z = Input(shape=(self.z_dim,), name='g_z_input')
if self.g_input == 'z':
g_input = z
elif self.g_input == 'x':
g_input = x
else:
g_input = Concatenate(axis=1, name='g_xz_input')([x, z])
# Hidden
hidden = Dense(n_hidden,
activation=h_activation,
name='g_hidden_relu')(g_input)
if batchnorm:
hidden = BatchNormalization(name='g_hidden_bn')(hidden)
perturb = Dense(self.x_dim,
activation=out_activation,
**regularizers,
name='g_perturb_sigmoid')(hidden)
# Dropout
perturb = Dropout(drop_rate, name='perturb_dropout')(perturb)
perturb = K.minimum(perturb,
1) # NB: dropout scales up the kept inputs,
# so clip to stay <=1 (for max later)..
# use K.clip Or K.minimum(perturb, 1)
# Output
x_adv = Maximum(name='g_adv_max')([perturb, x])
self.generator = Model([x, z], x_adv, name='Generator')
if summary:
self.generator.summary();
print()
# G parameters for logging
self.reg = get_reg_factors(regularizers)
self.g_log = {'in': self.g_input,
'h': f'({n_hidden},{h_activation})', 'bn': batchnorm,
'reg': self.reg, 'drop': drop_rate}
return self.generator
def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True):
conv_a = Conv2D(n_filts,
self._filt_size,
padding='same',
name=name_prefix + '_a',
**self._layer_kwargs)(last_layer)
conv_b = Conv2D(n_filts,
self._filt_size,
padding='same',
name=name_prefix + '_b',
**self._layer_kwargs)(last_layer)
last = Maximum(name=name_prefix + '_m')([conv_a, conv_b])
# pre-weights (i.e. post max), as per
# http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf
if dropout:
last = Dropout(self._dropout_p, name=name_prefix + '_d')(last)
return last
def build_network(self, X=None, y=None):
shape = (self.img_size[0], self.img_size[1], 3)
input_layer = Input(shape=shape, name='input')
last_ = input_layer
# Convolutional section
last_conv_per_level = []
sizes = []
for conv_level in range(self.conv_num + 1):
nfilters = self.conv_filter_num * 2**conv_level
sizes.append(nfilters)
# Convolutional layers
for c in range(self.conv_consecutive):
#cfs = max(2, self.conv_filter_size[0] - conv_level)
#cfs = (cfs, cfs)
cfs = self.conv_filter_size
last_ = Conv2D(nfilters,
cfs,
activation=self.conv_activation,
padding=self.conv_padding,
kernel_initializer=glorot_uniform(seed=42),
name='conv%d-%d' % (conv_level, c))(last_)
last_conv_per_level.append(last_)
# Pooling layer
if conv_level != self.conv_num:
last_ = MaxPooling2D(pool_size=self.pool_size,
strides=self.pool_strides,
name='pool%d' % conv_level)(last_)
last_per_label = [None]
if self.multiple_out_streams:
last_before = last_
for d in range(1, self.num_labels + 1):
last_ = last_before
# Deconvolutional section
for conv_level in range(self.conv_num)[::-1]:
# FIXME
#cc = Concatenate(axis=3)
cc = Add()
last_ = cc([
Conv2DTranspose(
sizes[conv_level],
self.pool_size,
strides=(self.pool_size, self.pool_size),
)(last_), last_conv_per_level[conv_level]
])
for c in range(self.conv_consecutive):
glorot = glorot_uniform(seed=42)
last_ = Conv2D(sizes[conv_level],
self.conv_filter_size,
activation=self.conv_activation,
kernel_initializer=glorot,
padding=self.conv_padding)(last_)
last_per_label.append(last_)
else:
# Deconvolutional section
for conv_level in range(self.conv_num)[::-1]:
# FIXME
#cc = Concatenate(axis=3)
cc = Add()
last_ = cc([
Conv2DTranspose(
sizes[conv_level],
self.pool_size,
strides=(self.pool_size, self.pool_size),
)(last_), last_conv_per_level[conv_level]
])
for c in range(self.conv_consecutive):
#cfs = max(2, self.conv_filter_size[0] - conv_level)
#cfs = (cfs, cfs)
cfs = self.conv_filter_size
last_ = Conv2D(sizes[conv_level],
cfs,
activation=self.conv_activation,
kernel_initializer=glorot_uniform(seed=42),
padding=self.conv_padding)(last_)
for d in range(1, self.num_labels + 1):
last_per_label.append(last_)
outputs = []
scoring_outputs = []
prob_outputs = []
reverse_layer = Lambda(lambda x: -x)
minimum_layer = \
lambda x, y: reverse_layer(Maximum()([reverse_layer(x),
reverse_layer(y)]))
scoring = Conv2D(1, (1, 1), activation='linear')(last_per_label[-1])
for d in range(1, self.num_labels + 1):
if self.pointwise_ordinal:
if d > 1:
if self.parallel_hyperplanes:
next_output = Conv2D(1, (1, 1),
activation='linear')(scoring)
else:
next_output = Conv2D(1, (1, 1), activation='linear')(
last_per_label[d])
next_output = Activation('sigmoid')(next_output)
scoring_outputs.append(next_output)
prob_outputs.append(next_output)
if self.pointwise_ordinal == 'min':
next_output = minimum_layer(outputs[-1], next_output)
elif self.pointwise_ordinal == 'mul':
next_output = Multiply()([outputs[-1], next_output])
else:
if self.parallel_hyperplanes:
next_output = Conv2D(1, (1, 1),
activation='linear')(scoring)
else:
next_output = Conv2D(1, (1, 1), activation='linear')(
last_per_label[d])
next_output = Activation('sigmoid',
name='sigmoid%d' % d)(next_output)
scoring_outputs.append(next_output)
prob_outputs.append(next_output)
else:
next_output = Conv2D(1, (1, 1),
activation='linear',
name='out%d' % d)(last_per_label[d])
next_output = Activation('sigmoid',
name='sigmoid%d' % d)(next_output)
scoring_outputs.append(next_output)
prob_outputs.append(next_output)
outputs.append(next_output)
out = Concatenate(axis=-1, name='output')(outputs)
subtract = lambda x, y: Add()([x, reverse_layer(y)])
elem_hinge = lambda x: Lambda(lambda k: K.maximum(k, 0.))(x)
sign = lambda x: Lambda(lambda k: K.sign(k))(x)
hor_monoticity_output = None
ver_monoticity_output = None
if self.include_distances:
dshape = (self.img_size[0], self.img_size[1], 1)
distance_input = Input(shape=dshape, name='dists')
w = 5
left_crop = Cropping2D(((0, 0), (w, 0)))
right_crop = Cropping2D(((0, 0), (0, w)))
top_crop = Cropping2D(((w, 0), (0, 0)))
bottom_crop = Cropping2D(((0, w), (0, 0)))
left_crop_dists = left_crop(distance_input)
right_crop_dists = right_crop(distance_input)
top_crop_dists = top_crop(distance_input)
bottom_crop_dists = bottom_crop(distance_input)
horizontal_dist_diff = sign(
subtract(left_crop_dists, right_crop_dists))
vertical_dist_diff = sign(
subtract(top_crop_dists, bottom_crop_dists))
hor_monoticity_outputs = []
ver_monoticity_outputs = []
for p in scoring_outputs:
left_crop_probs = left_crop(p)
right_crop_probs = right_crop(p)
top_crop_probs = top_crop(p)
bottom_crop_probs = bottom_crop(p)
horizontal_prob_diff = subtract(left_crop_probs,
right_crop_probs)
horizontal_monoticity = Multiply()(
[horizontal_dist_diff, horizontal_prob_diff])
vertical_prob_diff = subtract(top_crop_probs,
bottom_crop_probs)
vertical_monoticity = Multiply()(
[vertical_dist_diff, vertical_prob_diff])
horizontal_monoticity = elem_hinge(horizontal_monoticity)
vertical_monoticity = elem_hinge(vertical_monoticity)
hor_monoticity_outputs.append(horizontal_monoticity)
ver_monoticity_outputs.append(vertical_monoticity)
hor_monoticity_output = \
Add(name='hor-monoticity')(hor_monoticity_outputs)
ver_monoticity_output = \
Add(name='ver-monoticity')(ver_monoticity_outputs)
print len(prob_outputs)
"""
if self.num_labels > 2:
ord_regs = []
for d in range(self.num_labels - 1):
ordinal_diff = Add()([prob_outputs[d + 1],
Lambda(lambda x: -x)(prob_outputs[d])])
ordinal_reg = Lambda(lambda x: K.maximum(x, 0.))(ordinal_diff)
ord_regs.append(ordinal_reg)
ord_regs = Add(name='ord-reg')(ord_regs)
else:
ordinal_diff = Add()([prob_outputs[1],
Lambda(lambda x: -x)(prob_outputs[0])])
ord_regs = Lambda(lambda x: K.maximum(x, 0.),
name='ord-reg')(ordinal_diff)
"""
model_inputs = [input_layer]
model_outputs = [out]
loss = {'output': self.loss}
loss_weights = {'output': 1.}
if self.include_distances:
smoothness_weight = 0.5 * self.smoothness_weight
model_inputs.append(distance_input)
model_outputs.append(hor_monoticity_output)
model_outputs.append(ver_monoticity_output)
loss['hor-monoticity'] = mean_value_loss
loss['ver-monoticity'] = mean_value_loss
loss_weights['hor-monoticity'] = smoothness_weight
loss_weights['ver-monoticity'] = smoothness_weight
model = Model(inputs=[input_layer], outputs=[out])
opt_model = Model(inputs=model_inputs, outputs=model_outputs)
opt_model.compile(
'adadelta',
loss=loss,
loss_weights=loss_weights,
metrics={
'output': [worst_dice_coef, best_dice_coef],
},
)
self.final_shape = out._keras_shape
return opt_model, model
# for each member of support set
for tt in combinations(range(numsupportset), 2):
aa = siam([convolutionlayers[tt[0]], convolutionlayers[tt[1]]])
pairwiseinteractions[tt[0]].append(aa)
pairwiseinteractions[tt[1]].append(aa)
# Get Siamese interactions for query image
targetinteractions = []
for i in range(numsupportset):
aa = siam([targetembedding, convolutionlayers[i]])
targetinteractions.append(aa)
pairwiseinteractions[i].append(
aa) # add this interaction to the set of interaction for this member
# Take 4 layer MLP transform on Max pooling of interactions to serve as Full Context Embedding (FCE)
maxi = Maximum()
modelinputs = []
for i in range(numsupportset):
modelinputs.append(relationalembedding(maxi(pairwiseinteractions[i])))
modelinputs.append(relationalembedding(maxi(targetinteractions)))
supportlabels = Input((numsupportset, classes_per_set))
modelinputs.append(supportlabels)
knnsimilarity = MatchEuclidean(nway=classes_per_set)(modelinputs)
model = Model(inputs=[input1, supportlabels], outputs=knnsimilarity)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
def combined_model(self, batch_size, max_n_sentences, max_sentence_length,
max_word_length):
sentence_remainder, word_remainder = self.remainder_input_tensor(
max_n_sentences, max_sentence_length)
exists = Input(shape=(max_n_sentences, max_sentence_length),
dtype='float32',
name='exists')
known = Input(shape=(max_n_sentences, max_sentence_length),
dtype='float32',
name='known')
# embed word
word_input = Input(shape=(max_n_sentences, max_sentence_length),
dtype='int32',
name='doc-wd_in')
embedded_word = self.embed_word_document(word_input) # masked
# embed char
char_input = Input(shape=(max_n_sentences, max_sentence_length,
max_word_length),
dtype='int32',
name='doc_ch_in')
embedded_char = self.embed_char_document(char_input)
# combine (masked during merging)
if False:
# concat_word = Concatenate()([embedded_char, embedded_word])
# concat_word = self.conv_dense(concat_word)
embedded_word = self.max_relu(embedded_word)
embedded_char = self.max_relu(embedded_char)
concat_word = Maximum()([embedded_word, embedded_char])
# concat_word = self.max_tanh(concat_word)
elif self.mode == 'max':
# embedded_word = self.word_linear(embedded_word)
# embedded_char = self.char_linear(embedded_char)
concat_word = Maximum()([embedded_word, embedded_char])
# concat_word = self.max_tanh(concat_word)
elif self.mode == 'switch':
if True:
concat_word = Switch(self.word_embedding_dim, 4)(
[exists, known, embedded_word, embedded_char])
else:
concat_word = embedded_char
def expand(x):
y = K.expand_dims(exists)
return K.repeat_elements(y, self.word_embedding_dim, axis=3)
def calc_output_shape(in_shape):
return in_shape + (self.word_embedding_dim, )
expanded_exists = Lambda(expand,
output_shape=calc_output_shape)(exists)
concat_word = Multiply()([concat_word, expanded_exists])
# to document
output = self.embedded_word_to_document(max_sentence_length,
self.word_embedding_dim,
concat_word,
sentence_remainder,
word_remainder)
if not self.REMAINDER:
model = Model([exists, known, word_input, char_input], output)
else:
model = Model([
exists, known, word_input, char_input, sentence_remainder,
word_remainder
], output)
self.compile_model(model)
return model
counter += 1
if counter % 50 == 0:
print("i=" + str(counter))
if counter == int((1 - splits) * num_batches):
counter = 0
if shuffle:
np.random.shuffle(smp_idx)
nb_hidden = 1000
input1 = Input(shape=(1000, ))
input2 = Input(shape=(1000, ))
input3 = Input(shape=(1000, ))
avgL = Average()([input1, input2, input3])
maxL = Maximum()([input1, input2, input3])
minL = Minimum()([input1, input2, input3])
concatL = Concatenate()([avgL, maxL])
layer1 = Dense(nb_hidden,
kernel_regularizer=regularizers.l1(0.00001),
bias_initializer='zeros',
activation='relu')(concatL)
#layer1=Dropout(0.5)(layer1)
out = Dense(nb_classes,
kernel_regularizer=regularizers.l1(0.00001),
bias_initializer='zeros',
kernel_initializer='identity',
activation='softmax')(layer1)
model = Model([input1, input2, input3], out)
#model=load_model('genFit_ens450.h5');
def _construct_acoustic_model(self, train_config=TrainConfig()):
# construct an acoustic model from scratch
self._cur_weight_seed = self.config.weight_seed
def _layer_kwargs():
ret = {
'activation': 'linear',
'kernel_initializer': RandomUniform(
minval=-self.config.weight_init_mag,
maxval=self.config.weight_init_mag,
seed=self._cur_weight_seed,
),
}
self._cur_weight_seed = self._cur_weight_seed + 1
if train_config.training_stage == 'sgd':
ret['kernel_regularizer'] = l2(train_config.sgd_reg)
return ret
# convolutional layer pattern
def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True):
conv_a = Conv2D(
n_filts,
(self.config.filt_time_width, self.config.filt_freq_width),
strides=(
self.config.filt_time_stride,
self.config.filt_freq_stride,
),
padding='same',
name=name_prefix + '_a',
**_layer_kwargs()
)(last_layer)
conv_b = Conv2D(
n_filts,
(self.config.filt_time_width, self.config.filt_freq_width),
strides=(
self.config.filt_time_stride,
self.config.filt_freq_stride,
),
padding='same',
name=name_prefix + '_b',
**_layer_kwargs()
)(last_layer)
last = Maximum(name=name_prefix + '_m')([conv_a, conv_b])
# pre-weights (i.e. post max), as per
# http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf
if dropout:
last = Dropout(
train_config.dropout, name=name_prefix + '_d',
seed=self._cur_weight_seed)(last)
self._cur_weight_seed += 1
return last
# inputs
feat_input = Input(
shape=(
None,
self.config.num_feats * (1 + self.config.delta_order),
1,
),
name='feat_in',
)
feat_size_input = Input(
shape=(1,), dtype='int32', name='feat_size_in')
label_input = Input(
shape=(None,), dtype='int32', name='label_in')
label_size_input = Input(
shape=(1,), dtype='int32', name='label_size_in')
last_layer = feat_input
# convolutional layers
n_filts = self.config.init_num_filt_channels
last_layer = _conv_maxout_layer(
last_layer, n_filts, 'conv_1', dropout=False)
last_layer = MaxPooling2D(
pool_size=(
self.config.pool_time_width,
self.config.pool_freq_width),
name='conv_1_p')(last_layer)
last_layer = Dropout(
train_config.dropout, name='conv_1_d',
seed=self._cur_weight_seed)(last_layer)
self._cur_weight_seed += 1
for layer_no in range(2, 11):
if layer_no == 5:
n_filts *= 2
last_layer = _conv_maxout_layer(
last_layer, n_filts, 'conv_{}'.format(layer_no))
last_layer = Lambda(
lambda layer: K.max(layer, axis=2),
output_shape=(None, n_filts),
name='max_freq_into_channel',
)(last_layer)
# dense layers
for layer_no in range(1, 4):
name_prefix = 'dense_{}'.format(layer_no)
dense_a = Dense(
self.config.num_dense_hidden, name=name_prefix + '_a',
**_layer_kwargs()
)
dense_b = Dense(
self.config.num_dense_hidden, name=name_prefix + '_b',
**_layer_kwargs()
)
td_a = TimeDistributed(
dense_a, name=name_prefix + '_td_a')(last_layer)
td_b = TimeDistributed(
dense_b, name=name_prefix + '_td_b')(last_layer)
last_layer = Maximum(name=name_prefix + '_m')([td_a, td_b])
last_layer = Dropout(
train_config.dropout, name=name_prefix + '_d',
seed=self._cur_weight_seed,
)(last_layer)
self._cur_weight_seed += 1
activation_dense = Dense(
self.config.num_labels, name='dense_activation',
**_layer_kwargs()
)
activation_layer = TimeDistributed(
activation_dense, name='dense_activation_td')(last_layer)
# we take a page from the image_ocr example and treat the ctc as a
# lambda layer.
loss_layer = Lambda(
lambda args: _ctc_loss(*args),
output_shape=(1,), name='ctc_loss'
)([
label_input,
activation_layer,
feat_size_input,
label_size_input
])
self.model = Model(
inputs=[
feat_input,
label_input,
feat_size_input,
label_size_input,
],
outputs=[loss_layer],
)
def _construct_acoustic_model(self):
# construct acoustic model
# convolutional layer pattern
def _conv_maxout_layer(last_layer, n_filts, name_prefix, dropout=True):
conv_a = Conv2D(n_filts,
self._filt_size,
padding='same',
name=name_prefix + '_a',
**self._layer_kwargs)(last_layer)
conv_b = Conv2D(n_filts,
self._filt_size,
padding='same',
name=name_prefix + '_b',
**self._layer_kwargs)(last_layer)
last = Maximum(name=name_prefix + '_m')([conv_a, conv_b])
# pre-weights (i.e. post max), as per
# http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf
if dropout:
last = Dropout(self._dropout_p, name=name_prefix + '_d')(last)
return last
n_filts = self._initial_filts
# inputs
audio_input_shape = [self._input_shape[0], self._input_shape[1], 1]
if self._deltas:
if self._deltas.concatenate:
audio_input_shape[1] *= self._deltas.num_deltas + 1
else:
audio_input_shape[2] *= self._deltas.num_deltas + 1
self._audio_input = Input(shape=audio_input_shape, name='audio_in')
self._audio_size_input = Input(shape=(1, ), name='audio_size_in')
self._label_input = Input(shape=(None, ), name='label_in')
self._label_size_input = Input(shape=(1, ), name='label_size_in')
last_layer = self._audio_input
# convolutional layers
last_layer = _conv_maxout_layer(last_layer,
n_filts,
'conv_1',
dropout=False)
last_layer = MaxPooling2D(pool_size=self._pool_size,
name='conv_1_p')(last_layer)
last_layer = Dropout(self._dropout_p, name='conv_1_d')(last_layer)
for layer_no in range(2, 11):
if layer_no == 5:
n_filts *= 2
last_layer = _conv_maxout_layer(last_layer, n_filts,
'conv_{}'.format(layer_no))
last_layer = Lambda(
lambda layer: K.max(layer, axis=2),
output_shape=(
self._input_shape[0],
n_filts,
),
name='max_freq_into_channel',
)(last_layer)
# dense layers
for layer_no in range(1, 4):
name_prefix = 'dense_{}'.format(layer_no)
dense_a = Dense(self._dense_size,
name=name_prefix + '_a',
**self._layer_kwargs)
dense_b = Dense(self._dense_size,
name=name_prefix + '_b',
**self._layer_kwargs)
td_a = TimeDistributed(dense_a,
name=name_prefix + '_td_a')(last_layer)
td_b = TimeDistributed(dense_b,
name=name_prefix + '_td_b')(last_layer)
last_layer = Maximum(name=name_prefix + '_m')([td_a, td_b])
last_layer = Dropout(self._dropout_p,
name=name_prefix + '_d')(last_layer)
activation_dense = Dense(self._num_labels,
name='dense_activation',
**self._layer_kwargs)
self._activation_layer = TimeDistributed(
activation_dense, name='dense_activation_td')(last_layer)
# we take a page from the image_ocr example and treat the ctc as a
# lambda layer.
self._loss_layer = Lambda(lambda args: _ctc_loss(*args),
output_shape=(1, ),
name='ctc_loss')([
self._label_input,
self._activation_layer,
self._audio_size_input,
self._label_size_input
])
self._acoustic_model = Model(
inputs=[
self._audio_input,
self._label_input,
self._audio_size_input,
self._label_size_input,
],
outputs=[self._loss_layer],
)