def build_decoder_model_without_argmax(seq2seq, input_t, output_t):
# Remove all initializer.
input_state = Input(shape=(seq2seq.units, ), name="decoder_state")
decoder_inputs = Input(shape=(None, ), name="decoder_input")
decoder_embedding = Embedding(seq2seq.tgt_token_size,
seq2seq.units,
input_length=None,
name="decoder_emb")
decoder_gru = GRU(seq2seq.units,
return_sequences=True,
return_state=True,
name="decoder_gru")
decoder_dense = Dense(seq2seq.tgt_token_size,
activation="softmax",
name="output_dense")
state = input_state
for t in range(input_t, output_t):
inputs = Lambda(slice, arguments={"index": t})(
decoder_inputs) # Count encoder output as time 0.
inputs_embedding = decoder_embedding(inputs)
decoder_outputs_time, state = decoder_gru(inputs_embedding,
initial_state=state)
if input_t == output_t:
decoder_outputs_time = Lambda(lambda x: K.expand_dims(x, axis=1))(
state)
softmax = decoder_dense(decoder_outputs_time)
decoder_model = Model([decoder_inputs, input_state], [softmax] + [state])
return decoder_model
def Orthonorm(x, name=None):
'''
Builds keras layer that handles orthogonalization of x
x: an n x d input matrix
name: name of the keras layer
returns: a keras layer instance. during evaluation, the instance returns an n x d orthogonal matrix
if x is full rank and not singular
'''
# get dimensionality of x
d = x.get_shape().as_list()[-1]
# compute orthogonalizing matrix
ortho_weights = orthonorm_op(x)
# create variable that holds this matrix
ortho_weights_store = K.variable(np.zeros((d, d)))
# create op that saves matrix into variable
ortho_weights_update = tf.assign(ortho_weights_store,
ortho_weights,
name='ortho_weights_update')
# switch between stored and calculated weights based on training or validation
l = Lambda(lambda x: K.in_train_phase(K.dot(x, ortho_weights),
K.dot(x, ortho_weights_store)),
name=name)
l.add_update(ortho_weights_update)
return l
def __init__(self, inputs, arch, siam_reg, main_path, y_true):
self.orig_inputs = inputs
# set up inputs
self.inputs = {
'A': inputs['Unlabeled'],
'B': Input(shape=inputs['Unlabeled'].get_shape().as_list()[1:]),
'Labeled': inputs['Labeled'],
}
self.main_path = os.path.join(main_path, 'siemese/')
self.y_true = y_true
# generate layers
self.layers = []
self.layers += util.make_layer_list(arch, 'siamese', siam_reg)
# create the siamese net
self.outputs = stack_layers(self.inputs, self.layers)
# add the distance layer
self.distance = Lambda(affinities.euclidean_distance,
output_shape=affinities.eucl_dist_output_shape)(
[self.outputs['A'], self.outputs['B']])
# create the distance model for training
self.net = Model([self.inputs['A'], self.inputs['B']], self.distance)
# compile the siamese network
self.net.compile(loss=affinities.get_contrastive_loss(m_neg=1,
m_pos=0.05),
optimizer='rmsprop')
def build_GRU_with_z_gate_model(seq2seq, weight_array):
h_tm1_input = Input(shape=(seq2seq.units, ), name="h_input")
x_input = Input(shape=(seq2seq.units, ), name="x_input")
r_input = Input(shape=(seq2seq.units, ), name="r_input")
hh_input = Input(shape=(seq2seq.units, ), name="hh_input")
def gru_with_z_gate(x, weight):
h_tm1, inputs, r, hh = x[0], x[1], x[2], x[3]
weight = K.variable(weight)
units = h_tm1.shape[-1]
kernel_z = weight[:units, :units]
recurrent_kernel_z = weight[units:units * 2, :units]
input_bias_z = weight[units * 2, :units] # Change to 1 dim.
x_z = K.bias_add(K.dot(inputs, kernel_z), input_bias_z)
recurrent_z = K.dot(h_tm1, recurrent_kernel_z)
z_without_activate = x_z + recurrent_z
z = hard_sigmoid(z_without_activate)
h = z * h_tm1 + (1 - z) * hh
#return h
return z
output = Lambda(gru_with_z_gate,
arguments={"weight": weight_array
})([h_tm1_input, x_input, r_input, hh_input])
#h = layers.Add()([h_tm1_input, x_input])
gate_model = Model([h_tm1_input, x_input, r_input, hh_input], output)
#print("z gate model.")
#gate_model.summary()
return gate_model
def DNNclassifier_crps(self, p, num_cut, optimizer, seeding):
tf.set_random_seed(seeding)
inputs = Input(shape=(p,))
if isinstance(optimizer, str):
opt = optimizer
else:
opt_name = optimizer.__class__.__name__
opt_config = optimizer.get_config()
opt_class = getattr(optimizers, opt_name)
opt = opt_class(**opt_config)
for i, n_neuron in enumerate(self.hidden_list):
if i == 0:
net = Dense(n_neuron, kernel_initializer = 'he_uniform')(inputs)
else:
net = Dense(n_neuron, kernel_initializer = 'he_uniform')(net)
net = Activation(activation = 'elu')(net)
net = BatchNormalization()(net)
net = Dropout(rate=self.dropout_list[i])(net)
softmaxlayer = Dense(num_cut + 1, activation='softmax',
kernel_initializer = 'he_uniform')(net)
output = Lambda(self.tf_cumsum)(softmaxlayer)
model = Model(inputs = [inputs], outputs=[output])
model.compile(optimizer=opt, loss=self.crps_loss)
return model
def multi_gpu_model(model, gpus):
if isinstance(gpus, (list, tuple)):
num_gpus = len(gpus)
target_gpu_ids = gpus
else:
num_gpus = gpus
target_gpu_ids = range(num_gpus)
def get_slice(data, i, parts):
shape = tf.shape(data)
batch_size = shape[:1]
input_shape = shape[1:]
step = batch_size // parts
if i == num_gpus - 1:
size = batch_size - step * i
else:
size = step
size = tf.concat([size, input_shape], axis=0)
stride = tf.concat([step, input_shape * 0], axis=0)
start = stride * i
return tf.slice(data, start, size)
all_outputs = []
for i in range(len(model.outputs)):
all_outputs.append([])
# Place a copy of the model on each GPU,
# each getting a slice of the inputs.
for i, gpu_id in enumerate(target_gpu_ids):
with tf.device('/gpu:%d' % gpu_id):
with tf.name_scope('replica_%d' % gpu_id):
inputs = []
# Retrieve a slice of the input.
for x in model.inputs:
input_shape = tuple(x.get_shape().as_list())[1:]
slice_i = Lambda(get_slice,
output_shape=input_shape,
arguments={
'i': i,
'parts': num_gpus
})(x)
inputs.append(slice_i)
# Apply model on slice
# (creating a model replica on the target device).
outputs = model(inputs)
if not isinstance(outputs, list):
outputs = [outputs]
# Save the outputs for merging back together later.
for o in range(len(outputs)):
all_outputs[o].append(outputs[o])
# Merge outputs on CPU.
with tf.device('/cpu:0'):
merged = []
for name, outputs in zip(model.output_names, all_outputs):
merged.append(concatenate(outputs, axis=0, name=name))
return Model(model.inputs, merged)
def build_fully_conv(obs_spec,
act_spec,
data_format='channels_first',
broadcast_non_spatial=False,
fc_dim=256):
screen, screen_input = spatial_block('screen', obs_spec.spaces[0],
conv_cfg(data_format, 'relu'))
minimap, minimap_input = spatial_block('minimap', obs_spec.spaces[1],
conv_cfg(data_format, 'relu'))
non_spatial_inputs = [Input(s.shape) for s in obs_spec.spaces[2:]]
if broadcast_non_spatial:
non_spatial, spatial_dim = non_spatial_inputs[1], obs_spec.spaces[
0].shape[1]
non_spatial = tf.log(non_spatial + 1e-5)
broadcasted_non_spatial = Broadcast2D(spatial_dim)(non_spatial)
state = tf.concat([screen, minimap, broadcasted_non_spatial], axis=1)
else:
state = tf.concat([screen, minimap], axis=1)
fc = Flatten(name="state_flat")(state)
fc = Dense(fc_dim, **dense_cfg('relu'))(fc)
value = Dense(1, name="value_out", **dense_cfg(scale=0.1))(fc)
value = tf.squeeze(value, axis=-1)
logits = []
for space in act_spec:
if space.is_spatial():
logits.append(
Conv2D(1, 1, **conv_cfg(data_format, scale=0.1))(state))
logits[-1] = Flatten()(logits[-1])
else:
logits.append(Dense(space.size(), **dense_cfg(scale=0.1))(fc))
mask_actions = Lambda(lambda x: tf.where(non_spatial_inputs[0] > 0, x,
-1000 * tf.ones_like(x)),
name="mask_unavailable_action_ids")
logits[0] = mask_actions(logits[0])
return Model(inputs=[screen_input, minimap_input] + non_spatial_inputs,
outputs=logits + [value])
def build_GRU_with_r_gate_model(seq2seq, weight_array):
h_tm1_input = Input(shape=(seq2seq.units, ), name="h_input")
x_input = Input(shape=(seq2seq.units, ), name="x_input")
z_input = Input(shape=(seq2seq.units, ), name="z_input")
xh_input = Input(
shape=(seq2seq.units, ), name="xh_input"
) # x_h = K.bias_add(K.dot(inputs, kernel_h), input_bias_h)
rh_input = Input(
shape=(seq2seq.units, seq2seq.units), name="rh_input"
) # split_recurrent_h = K.dot(h_tm1.transpose(), recurrent_kernel_h)
def gru_with_r_gate(x, weight):
h_tm1, inputs, z, x_h, split_recurrent_h = x[0], x[1], x[2], x[3], x[4]
weight = K.variable(weight)
units = h_tm1.shape[-1]
kernel_r = weight[:units, units:units * 2]
recurrent_kernel_r = weight[units:units * 2, units:units * 2]
input_bias_r = weight[units * 2, units:units * 2] # Change to 1 dim.
x_r = K.bias_add(K.dot(inputs, kernel_r), input_bias_r)
recurrent_r = K.dot(h_tm1, recurrent_kernel_r)
r_without_activate = x_r + recurrent_r
r = hard_sigmoid(r_without_activate)
#r = hard_sigmoid(x_r + recurrent_r)
# Recompute recurrent_h by two parts.
r_unsqueeze = K.expand_dims(r, axis=-1)
recompute_recurrent_h = K.sum(r_unsqueeze * split_recurrent_h, axis=1)
hh = tanh(x_h + recompute_recurrent_h)
h = z * h_tm1 + (1 - z) * hh
#return h
return r
output = Lambda(gru_with_r_gate, arguments={"weight": weight_array})(
[h_tm1_input, x_input, z_input, xh_input, rh_input])
gate_model = Model([h_tm1_input, x_input, z_input, xh_input, rh_input],
output)
#print("r gate model")
#gate_model.summary()
return gate_model
def seq2seq(src_max_len,
tgt_max_len,
src_token_size,
tgt_token_size,
latent_dim=128,
teacher_forcing_ratio=0.5):
rd = RandomUniform(minval=-0.08, maxval=0.08, seed=None)
encoder_inputs = Input(shape=(None, ), name='encoder_inputs')
print('(Build model) encoder_inputs =', encoder_inputs.shape)
encoder_embedding = Embedding(src_token_size,
latent_dim,
embeddings_initializer=rd,
input_length=None,
mask_zero=True,
name='encoder_emb')(encoder_inputs)
print('(Build model) encoder_embedding =', encoder_embedding.shape)
encoder_time, encoder_state_h = GRU(latent_dim,
kernel_initializer=rd,
bias_initializer=rd,
return_state=True,
return_sequences=True,
name='forward')(encoder_embedding)
print("(Build model) encoder_state_h =", encoder_state_h.shape)
encoder_model = Model(encoder_inputs, [encoder_state_h, encoder_time])
decoder_inputs = Input(shape=(None, ), name='decoder_inputs')
print('(Build model) decoder_inputs =', decoder_inputs.shape)
decoder_embedding = Embedding(tgt_token_size,
latent_dim,
embeddings_initializer=rd,
input_length=None,
name='decoder_emb')
decoder_gru = GRU(latent_dim,
kernel_initializer=rd,
bias_initializer=rd,
return_sequences=True,
return_state=True,
name='decoder_gru')
decoder_dense = Dense(tgt_token_size,
kernel_initializer=rd,
bias_initializer=rd,
activation='softmax',
name='output_dense')
inputs = Lambda(slice, arguments={'h1': 0})(decoder_inputs)
softmax_state = []
teacher_forcing = Input(shape=(None, ), )
decoder_state_h = encoder_state_h
# Run decoder on each timestep.
for i in range(tgt_max_len):
inputs_embed = decoder_embedding(inputs)
decoder_outputs_time, state_h = decoder_gru(
inputs_embed, initial_state=decoder_state_h)
softmax = decoder_dense(decoder_outputs_time)
outputs = Lambda(lambda x: K.argmax(x))(softmax)
outputs = Lambda(lambda x: K.cast(outputs, 'float32'))(outputs)
decoder_inputs_time = Lambda(slice, arguments={'h1':
i + 1})(decoder_inputs)
inputs = Lambda(where, arguments={'ratio': teacher_forcing_ratio})(
[teacher_forcing, decoder_inputs_time, outputs])
# inputs = Lambda(where, arguments={'ratio': 0.5})([teacher_forcing, outputs, decoder_inputs_time])
decoder_state_h = state_h
softmax_state += [softmax]
decoder_outputs = Lambda(lambda x: K.concatenate(x, axis=1))(softmax_state)
# Define the model that will turn "encoder_input_data" & "decoder_input_data" into "decoder_target_data".
model = Model([encoder_inputs, decoder_inputs, teacher_forcing],
decoder_outputs)
# model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
# model.summary()
decoder_state_input_h = Input(shape=(latent_dim, ), name='decoder_input_h')
decoder_outputs, state_h = decoder_gru(decoder_embedding(decoder_inputs),
initial_state=decoder_state_input_h)
print('(Build model) decoder_outputs =', decoder_outputs)
decoder_outputs = decoder_dense(decoder_outputs)
print('(Build model) decoder_outputs =', decoder_outputs)
decoder_model = Model([decoder_inputs] + [decoder_state_input_h],
[decoder_outputs] + [state_h])
encoder_model.summary()
decoder_model.summary()
return model, encoder_model, decoder_model
def insert_layer_old(model, key_layer_name):
"""
Insert a new layer right after the layer specified in the argument.
Parameters
----------
model : Keras Model
The original model
key_layer_name : str
Name of the layer after which the new one is inserted.
Returns
-------
new_model : Keras Model
The modified new model
"""
# Determine output layer of the model
output_layer = model.get_layer('softmax')
# Extract information from the key key layer, after which the new layer
# will be inserted
key_layer = model.get_layer(key_layer_name)
x = key_layer.output
# Determine next layer after the key layer
next_layer = key_layer._outbound_nodes[0].outbound_layer
key_layer._outbound_nodes = []
# Attach the new layers
n_bn = 1
for n in range(n_bn):
alpha = 1e9
# beta = 1.
# beta = 1000.
beta = 0.
# x = Lambda(lambda x: alpha * x + beta)(x)
# x = Lambda(lambda x: alpha * (x - K.min(x)) / (K.max(x) - K.min(x)))(x)
# x = Lambda(lambda x: K.max(x))(x)
temp = 0.001
x = Lambda(lambda x: x / temp)(x)
# x = Lambda(lambda x: alpha * K.batch_normalization(x, mean=K.mean(x, axis=0), var=K.var(x, axis=0), beta=0., gamma=1.))(x)
# x = multiply([x, x], name='multiply{}'.format(n+1))
# x = add([x, x], name='add{}'.format(n+1))
# Connect the new layers to the original 'next_layer'
next_layer._inbound_nodes = []
x = next_layer(x)
# Re-connect the rest of the layers
while next_layer is not output_layer:
next_layer = next_layer._outbound_nodes[0].outbound_layer
next_layer._inbound_nodes[0].inbound_layers[0]._outbound_nodes = []
next_layer._inbound_nodes = []
x = next_layer(x)
new_model = Model(inputs=model.input, outputs=x)
return new_model
def __init__(self, size):
Lambda.__init__(self, lambda x: tf.tile(tf.expand_dims(tf.expand_dims(x, 2), 3), [1, 1, size, size]))
def __init__(self, scale):
Lambda.__init__(self, lambda x: tf.cast(x, tf.float32) * scale)
def __init__(self):
Lambda.__init__(self, lambda x: tf.log(x + 1e-10))
def __init__(self, dims):
Lambda.__init__(self, lambda x: tf.transpose(x, dims))
def __init__(self, num_splits=2, axis=-1, name=None):
Lambda.__init__(self, lambda x: tf.split(x, num_splits, axis=axis), name=name)
def __init__(self, axis=-1, name=None):
Lambda.__init__(self, lambda x: tf.squeeze(x, axis=axis), name=name)