def fully_connected(features, mode):
keep_prob = hypers.get_param('fd')
training = (mode == tf.contrib.learn.ModeKeys.TRAIN)
fc_act_func = activationFunc(hypers.get_param('fa'))
width = hypers.get_param('fw')
bias = hypers.get_param('fb')
scale = hypers.get_param('ff')
mode = hypers.get_param('fm')
uniform = hypers.get_param('fu')
if width > 0:
winitializer = ctblayers.variance_scaling_initializer(factor=scale,
mode=mode,
uniform=uniform)
binitializer = tf.constant_initializer(bias)
output = ctblayers.fully_connected(features,
width,
weights_initializer=winitializer,
biases_initializer=binitializer,
activation_fn=fc_act_func)
else:
output = features
if keep_prob > 0.0 and keep_prob < 1.0:
output = tf.layers.dropout(output, keep_prob, training=training)
# [dim] output fc fw>0 [bc, fw]
# [dim] output fc fw=0 [bc, input]
return output
def layer_normalize(features, mode):
act_func = activationFunc(hypers.get_param('wa'))
center = hypers.get_param('wc')
scale = hypers.get_param('w1')
output = ctblayers.layer_norm(features,
center=center,
scale=scale,
activation_fn=act_func)
# dim output normalize ['tb', 'dx', 'ws']
return output
def output(features):
width = hypers.get_param('on')
act_func = activationFunc(hypers.get_param('oa'))
bias = hypers.get_param('ob')
scale = hypers.get_param('of')
mode = hypers.get_param('om')
uniform = hypers.get_param('ou')
winitializer = ctblayers.variance_scaling_initializer(factor=scale, mode=mode, uniform=uniform)
binitializer = tf.constant_initializer(bias)
regularizer = ctblayers.l1_l2_regularizer(scale_l1=hypers.get_param('o1'), scale_l2=hypers.get_param('o2'))
out_layer = ctblayers.fully_connected(features, width, weights_initializer=winitializer, weights_regularizer=regularizer, biases_initializer=binitializer, biases_regularizer=regularizer, activation_fn=act_func)
# [dim] output ot fw>0 [bc, fw]
return out_layer
def batch_normalize(features, mode):
act_func = activationFunc(hypers.get_param('wa'))
decay = hypers.get_param('wy')
center = hypers.get_param('wc')
scale = hypers.get_param('w1')
epsilon = hypers.get_param('we')
training = (mode == tf.contrib.learn.ModeKeys.TRAIN)
output = ctblayers.batch_norm(features,
decay=decay,
center=center,
scale=scale,
epsilon=epsilon,
activation_fn=act_func,
is_training=training)
# dim output normalize ['tb', 'dx', 'ws']
return output
def rnn_cells():
cell_type = hypers.get_param('rc')
act_func = activationFunc(hypers.get_param('ra'))
forget_bias = hypers.get_param('rb')
rnn_width = hypers.get_param('rw')
peepholes = hypers.get_param('rp')
# Forward / Backward layer_type cell
if cell_type == "lstm":
fw_cell = lstm_cell(rnn_width,
use_peepholes=peepholes,
forget_bias=forget_bias,
activation=act_func)
bw_cell = lstm_cell(rnn_width,
use_peepholes=peepholes,
forget_bias=forget_bias,
activation=act_func)
elif cell_type == "clstm":
fw_cell = clstm_cell(rnn_width,
use_peepholes=peepholes,
forget_bias=forget_bias,
activation=act_func)
bw_cell = clstm_cell(rnn_width,
use_peepholes=peepholes,
forget_bias=forget_bias,
activation=act_func)
elif cell_type == "gru":
fw_cell = gru_cell(rnn_width, activation=act_func)
bw_cell = gru_cell(rnn_width, activation=act_func)
elif cell_type == "ugrnn":
fw_cell = ugrnn_cell(rnn_width,
forget_bias=forget_bias,
activation=act_func)
bw_cell = ugrnn_cell(rnn_width,
forget_bias=forget_bias,
activation=act_func)
return fw_cell, bw_cell