def single_cell(cell_class,
cell_params,
dp_input_keep_prob=1.0,
dp_output_keep_prob=1.0,
residual_connections=False):
"""Creates an instance of the rnn cell.
Such cell describes one step one layer and can include residual connection
and/or dropout
Args:
cell_class: Tensorflow RNN cell class
cell_params (dict): cell parameters
dp_input_keep_prob (float): (default: 1.0) input dropout keep probability
dp_output_keep_prob (float): (default: 1.0) output dropout keep probability
residual_connections (bool): whether to add residual connection
Returns:
TF RNN instance
"""
cell = cell_class(**cell_params)
if residual_connections:
cell = ResidualWrapper(cell)
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0:
cell = DropoutWrapper(cell,
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
return cell
def single_cell(
cell_class,
cell_params,
dp_input_keep_prob=1.0,
dp_output_keep_prob=1.0,
recurrent_keep_prob=1.0,
input_weight_keep_prob=1.0,
recurrent_weight_keep_prob=1.0,
weight_variational=False,
dropout_seed=None,
zoneout_prob=0.0,
training=True,
residual_connections=False,
awd_initializer=False,
variational_recurrent=False, # in case they want to use DropoutWrapper
dtype=None,
):
"""Creates an instance of the rnn cell.
Such cell describes one step one layer and can include residual connection
and/or dropout
Args:
cell_class: Tensorflow RNN cell class
cell_params (dict): cell parameters
dp_input_keep_prob (float): (default: 1.0) input dropout keep
probability.
dp_output_keep_prob (float): (default: 1.0) output dropout keep
probability.
zoneout_prob(float): zoneout probability. Applying both zoneout and
droupout is currently not supported
residual_connections (bool): whether to add residual connection
Returns:
TF RNN instance
"""
if awd_initializer:
val = 1.0 / math.sqrt(cell_params['num_units'])
cell_params['initializer'] = tf.random_uniform_initializer(minval=-val,
maxval=val)
cell = cell_class(**cell_params)
if residual_connections:
cell = ResidualWrapper(cell)
if zoneout_prob > 0.0 and (dp_input_keep_prob < 1.0
or dp_output_keep_prob < 1.0):
raise ValueError(
'Currently applying both dropout and zoneout on the same cell.'
'This is currently not supported.')
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0 and training:
cell = DropoutWrapper(
cell,
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob,
variational_recurrent=variational_recurrent,
dtype=dtype,
seed=dropout_seed,
)
if zoneout_prob > 0.0:
cell = ZoneoutWrapper(cell, zoneout_prob, is_training=training)
return cell
def _create_single_rnn_cell(self, num_units):
cell = GRUCell(
num_units) if self.cfg["cell_type"] == "gru" else LSTMCell(
num_units)
if self.cfg["use_dropout"]:
cell = DropoutWrapper(cell, output_keep_prob=self.rnn_keep_prob)
if self.cfg["use_residual"]:
cell = ResidualWrapper(cell)
return cell
def _create_rnn_cell(self):
cell = GRUCell(
self.cfg.num_units) if self.cfg.cell_type == "gru" else LSTMCell(
self.cfg.num_units)
if self.cfg.use_dropout:
cell = DropoutWrapper(cell, output_keep_prob=self.keep_prob)
if self.cfg.use_residual:
cell = ResidualWrapper(cell)
return cell
def setup_cell(cell_type, size, use_residual=False, keep_prob=None):
cell = getattr(tf.contrib.rnn, cell_type)(size)
if keep_prob is not None:
cell = DropoutWrapper(cell,
input_keep_prob=1.0,
output_keep_prob=keep_prob)
if use_residual:
cell = ResidualWrapper(cell)
return cell
def single_cell(cell_params):
# TODO: This method is ugly - redo
size = cell_params["num_units"]
if cell_type == "lstm":
cell_class = LSTMCell
elif cell_type == "gru":
cell_class = GRUCell
elif cell_type == "glstm":
cell_class = GLSTMCell
num_groups = 4#cell_params["num_groups"]
if residual_connections:
if dp_input_keep_prob !=1.0 or dp_output_keep_prob != 1.0:
if cell_type != "glstm":
return DropoutWrapper(ResidualWrapper(cell_class(num_units=size)),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
return DropoutWrapper(ResidualWrapper(cell_class(num_units=size, number_of_groups=num_groups)),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
if cell_type != "glstm":
return ResidualWrapper(cell_class(num_units=size))
else:
return ResidualWrapper(cell_class(num_units=size, number_of_groups=num_groups))
else:
if dp_input_keep_prob !=1.0 or dp_output_keep_prob != 1.0:
if cell_type != "glstm":
return DropoutWrapper(cell_class(num_units=size),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
return DropoutWrapper(cell_class(num_units=size, number_of_groups=num_groups),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
if cell_type != "glstm":
return cell_class(num_units=size)
else:
return cell_class(num_units=size, number_of_groups=num_groups)
def build_decoder_cell(rank, u_emb, batch_size, depth=2):
cell = []
for i in range(depth):
if i == 0:
cell.append(LSTMCell(rank, state_is_tuple=True))
else:
cell.append(ResidualWrapper(LSTMCell(rank, state_is_tuple=True)))
initial_state = LSTMStateTuple(tf.zeros_like(u_emb), u_emb)
initial_state = [initial_state, ]
for i in range(1, depth):
initial_state.append(cell[i].zero_state(batch_size, tf.float32))
return MultiRNNCell(cell), tuple(initial_state)
def single_cell(num_units):
if cell_type == "lstm":
cell_class = LSTMCell
elif cell_type == "gru":
cell_class = GRUCell
if residual_connections:
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0:
return DropoutWrapper(ResidualWrapper(
cell_class(num_units=num_units)),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
return ResidualWrapper(cell_class(num_units=num_units))
else:
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0:
return DropoutWrapper(cell_class(num_units=num_units),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
return cell_class(num_units=num_units)
def build_single_cell(self):
cell_type = LSTMCell
if (self.cell_type.lower() == 'gru'):
cell_type = GRUCell
cell = cell_type(self.hidden_units)
if self.use_dropout:
cell = DropoutWrapper(cell, dtype=self.dtype,
output_keep_prob=self.keep_prob_placeholder,)
if self.use_residual:
cell = ResidualWrapper(cell)
return cell
def build_single_cell(self, hidden_size=None):
if (self.p.cell_type.lower() == 'gru'): cell_type = GRUCell
else: cell_type = LSTMCell
if not hidden_size: hidden_size = self.p.hidden_size
cell = cell_type(hidden_size)
if self.p.use_dropout:
cell = DropoutWrapper(
cell, dtype=self.p.dtype,
output_keep_prob=self.p.keep_prob) #change this
if self.p.use_residual: cell = ResidualWrapper(cell)
return cell
def _build_single_cell(self):
if self.cell_type == 'lstm':
cell_type = LSTMCell
elif self.cell_type == 'gru':
cell_type = GRUCell
cell = cell_type(self.hidden_dim)
if self.use_dropout:
cell = DropoutWrapper(
cell,
dtype=self.dtype,
output_keep_prob=self.keep_prob_placeholder,
)
if self.use_residual:
cell = ResidualWrapper(cell)
return cell
def single_cell(cell_class,
cell_params,
dp_input_keep_prob=1.0,
dp_output_keep_prob=1.0,
zoneout_prob=0.,
training=True,
residual_connections=False):
"""Creates an instance of the rnn cell.
Such cell describes one step one layer and can include residual connection
and/or dropout
Args:
cell_class: Tensorflow RNN cell class
cell_params (dict): cell parameters
dp_input_keep_prob (float): (default: 1.0) input dropout keep
probability.
dp_output_keep_prob (float): (default: 1.0) output dropout keep
probability.
zoneout_prob(float): zoneout probability. Applying both zoneout and
droupout is currently not supported
residual_connections (bool): whether to add residual connection
Returns:
TF RNN instance
"""
cell = cell_class(**cell_params)
if residual_connections:
cell = ResidualWrapper(cell)
if zoneout_prob > 0. and (dp_input_keep_prob < 1.0
or dp_output_keep_prob < 1.0):
raise ValueError(
"Currently applying both dropout and zoneout on the same cell."
"This is not recommended")
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0:
cell = DropoutWrapper(cell,
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
if zoneout_prob > 0.:
cell = ZoneoutWrapper(cell, zoneout_prob, is_training=training)
return cell
def single_cell(cell_params):
# TODO: This method is ugly - redo
size = cell_params["num_units"]
proj_size = None if "proj_size" not in cell_params else cell_params[
"proj_size"]
if cell_type == "lstm":
if not residual_connections:
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return tf.nn.rnn_cell.LSTMCell(num_units=size,
num_proj=proj_size,
forget_bias=1.0)
else:
return DropoutWrapper(tf.nn.rnn_cell.LSTMCell(
num_units=size, num_proj=proj_size, forget_bias=1.0),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else: # residual connection required
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return ResidualWrapper(
tf.nn.rnn_cell.LSTMCell(num_units=size,
num_proj=proj_size,
forget_bias=1.0))
else:
return ResidualWrapper(
DropoutWrapper(
tf.nn.rnn_cell.LSTMCell(
num_units=size,
num_proj=proj_size,
forget_bias=1.0,
),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob,
))
elif cell_type == "gru":
if not residual_connections:
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return tf.nn.rnn_cell.GRUCell(num_units=size)
else:
return DropoutWrapper(
tf.nn.rnn_cell.GRUCell(num_units=size),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob,
)
else: # residual connection required
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return ResidualWrapper(
tf.nn.rnn_cell.GRUCell(num_units=size))
else:
return ResidualWrapper(
DropoutWrapper(tf.nn.rnn_cell.GRUCell(num_units=size),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob), )
elif cell_type == "glstm":
num_groups = cell_params["num_groups"]
if not residual_connections:
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return GLSTMCell(num_units=size,
number_of_groups=num_groups,
num_proj=proj_size,
forget_bias=1.0)
else:
return DropoutWrapper(GLSTMCell(
num_units=size,
number_of_groups=num_groups,
num_proj=proj_size,
forget_bias=1.0),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else: # residual connection required
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return ResidualWrapper(
GLSTMCell(num_units=size,
number_of_groups=num_groups,
num_proj=proj_size,
forget_bias=1.0))
else:
return ResidualWrapper(
DropoutWrapper(
GLSTMCell(
num_units=size,
number_of_groups=num_groups,
num_proj=proj_size,
forget_bias=1.0,
),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob,
))
elif cell_type == "slstm":
if not residual_connections:
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return BasicSLSTMCell(num_units=size)
else:
return DropoutWrapper(BasicSLSTMCell(num_units=size),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else: # residual connection required
if dp_input_keep_prob == 1.0 and dp_output_keep_prob == 1.0:
return ResidualWrapper(BasicSLSTMCell(num_units=size))
else:
return ResidualWrapper(
DropoutWrapper(
BasicSLSTMCell(num_units=size),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob,
))
else:
raise ValueError("Unknown RNN cell class: {}".format(cell_type))
# x_data = np.array([[[1,0,0,0]]],dtype = np.float32)
#
# outputs, _state = tf.nn.dynamic_rnn(cell1, x_data, dtype = tf.float32)
sess.run(tf.global_variables_initializer())
n_layers = 3
cells = []
for i in range(n_layers):
cell = tf.contrib.rnn.LSTMCell(hidden_size, state_is_tuple=True)
cell = tf.contrib.rnn.AttentionCellWrapper(cell,
attn_length=40,
state_is_tuple=True)
cell = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob=0.5)
# cell = tf.contrib.rnn.ResidualWrapper(cell)
cell = ResidualWrapper(cell)
cells.append(cell)
cell = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True)
def build_encoder():
print("building encoder..")
with tf.variable_scope('encoder'):
outputs, last_state = tf.nn.dynamic_rnn(cell=cell,
inputs=X_data,
dtype=tf.float32)
build_encoder()