def bidirectional_dynamic_rnn(cell_fw,
cell_bw,
inputs,
sequence_length=None,
initial_state_fw=None,
initial_state_bw=None,
dtype=None,
parallel_iterations=None,
swap_memory=False,
time_major=False,
scope=None):
assert not time_major
flat_inputs = flatten(inputs, 2) # [-1, J, d]
flat_len = None if sequence_length is None else tf.cast(
flatten(sequence_length, 0), 'int64')
(flat_fw_outputs, flat_bw_outputs), final_state = \
_bidirectional_dynamic_rnn(cell_fw, cell_bw, flat_inputs, sequence_length=flat_len,
initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw,
dtype=dtype, parallel_iterations=parallel_iterations, swap_memory=swap_memory,
time_major=time_major, scope=scope)
fw_outputs = reconstruct(flat_fw_outputs, inputs, 2)
bw_outputs = reconstruct(flat_bw_outputs, inputs, 2)
# FIXME : final state is not reshaped!
return (fw_outputs, bw_outputs), final_state
def dynamic_rnn(cell,
inputs,
sequence_length=None,
initial_state=None,
dtype=None,
parallel_iterations=None,
swap_memory=False,
time_major=False,
scope=None):
assert not time_major # TODO : to be implemented later!
flat_inputs = flatten(inputs, 2) # [-1, J, d]
flat_len = None if sequence_length is None else tf.cast(
flatten(sequence_length, 0), 'int64')
flat_outputs, final_state = _dynamic_rnn(
cell,
flat_inputs,
sequence_length=flat_len,
initial_state=initial_state,
dtype=dtype,
parallel_iterations=parallel_iterations,
swap_memory=swap_memory,
time_major=time_major,
scope=scope)
outputs = reconstruct(flat_outputs, inputs, 2)
return outputs, final_state
def __init__(self,
cell,
memory,
mask=None,
controller=None,
mapper=None,
input_keep_prob=1.0,
is_train=None):
"""
Early fusion attention cell: uses the (inputs, state) to control the current attention.
:param cell:
:param memory: [N, M, m]
:param mask:
:param controller: (inputs, prev_state, memory) -> memory_logits
"""
self._cell = cell
self._memory = memory
self._mask = mask
self._flat_memory = flatten(memory, 2)
self._flat_mask = flatten(mask, 1)
if controller is None:
controller = AttentionCell.get_linear_controller(True,
is_train=is_train)
self._controller = controller
if mapper is None:
mapper = AttentionCell.get_concat_mapper()
elif mapper == 'sim':
mapper = AttentionCell.get_sim_mapper()
self._mapper = mapper
def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None,
initial_state_fw=None, initial_state_bw=None,
dtype=None, parallel_iterations=None,
swap_memory=False, time_major=False, scope=None):
assert not time_major
flat_inputs = flatten(inputs, 2) # [-1, J, d]
#flat_inputs = flatten2(inputs) # [-1, J, d]
#tmpshape = tf.shape(inputs)
# flat_inputs = tf.reshape(inputs,(tmpshape[0],-1,tmpshape[-1]))
#flat_inputs = tf.reshape(inputs, (tmpshape[0], , tmpshape[len(tmpshape)-1]))
flat_len = None if sequence_length is None else tf.cast(flatten(sequence_length, 0), 'int64')
print ("inputs==={} ||| flat_inputs==={} ||| flat_len:{}".format(inputs,flat_inputs,flat_len))
# (flat_fw_outputs, flat_bw_outputs), final_state = \
# tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, flat_inputs, sequence_length=flat_len,
# initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw,
# dtype=dtype, parallel_iterations=parallel_iterations, swap_memory=swap_memory,
# time_major=time_major, scope=scope)
(flat_fw_outputs, flat_bw_outputs), final_state = \
_bidirectional_dynamic_rnn(cell_fw, cell_bw, flat_inputs, sequence_length=flat_len,
initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw,
dtype=dtype, parallel_iterations=parallel_iterations, swap_memory=swap_memory,
time_major=time_major, scope=scope)
fw_outputs = reconstruct(flat_fw_outputs, inputs, 2)
bw_outputs = reconstruct(flat_bw_outputs, inputs, 2)
# FIXME : final state is not reshaped!
return (fw_outputs, bw_outputs), final_state
def linear(args,
output_size,
bias,
bias_start=0.0,
scope=None,
squeeze=False,
wd=0.0,
input_keep_prob=1.0):
if args is None or (nest.is_sequence(args) and not args):
raise ValueError("`args` must be specified")
if not nest.is_sequence(args):
args = [args]
flat_args = [flatten(arg, 1) for arg in args]
if input_keep_prob < 1.0:
flat_args = [tf.nn.dropout(arg, input_keep_prob) for arg in flat_args]
#flat_out = _linear(flat_args, output_size, bias, bias_start=bias_start, scope=scope)
flat_out = tf.layers.dense(flat_args[0], output_size, use_bias=bias)
out = reconstruct(flat_out, args[0], 1)
if squeeze:
out = tf.squeeze(out, [len(args[0].get_shape().as_list()) - 1])
if wd:
add_wd(wd)
return out
def linear(args,
output_size,
bias,
bias_start=0.0,
scope=None,
squeeze=False,
wd=0.0,
input_keep_prob=1.0,
is_train=None):
if args is None or (nest.is_sequence(args) and not args):
raise ValueError("`args` must be specified")
if not nest.is_sequence(args):
args = [args]
flat_args = [flatten(arg, 1) for arg in args]
if input_keep_prob < 1.0:
assert is_train is not None
flat_args = [
tf.cond(is_train, lambda: tf.nn.dropout(arg, input_keep_prob),
lambda: arg) for arg in flat_args
]
flat_out = _linear(flat_args, output_size, bias)
out = reconstruct(flat_out, args[0], 1)
if squeeze:
out = tf.squeeze(out, [len(args[0].get_shape().as_list()) - 1])
if wd:
add_wd(wd)
return out
def softmax(logits, mask=None, scope=None):
with tf.name_scope(scope or "Softmax"):
if mask is not None:
logits = exp_mask(logits, mask)
flat_logits = flatten(logits, 1)
flat_out = tf.nn.softmax(flat_logits)
out = reconstruct(flat_out, logits, 1)
return out
def self_attention(self, x, is_train=True, squeeze=False):
with tf.variable_scope("attention"):
l_x = x.shape[1]
x_aug_1 = tf.tile(tf.expand_dims(x, axis=2), [1, 1, l_x, 1])
x_aug_2 = tf.tile(tf.expand_dims(x, axis=1), [1, l_x, 1, 1])
new_x = x_aug_1 * x_aug_2
flat_args = [x_aug_1, x_aug_2, new_x]
flat_args = [flatten(arg, 1) for arg in flat_args]
flat_args = [tf.cond(is_train, lambda: self.dropout(arg), lambda: arg) for
arg in flat_args]
flat_out = _linear(flat_args, 1, False)
out = reconstruct(flat_out, x_aug_1, 1)
if squeeze:
out = tf.squeeze(out, [len(x_aug_1.get_shape().as_list()) - 1])
return out
def run(self):
"""Run the simulation"""
if self.exe is None:
raise SimulationError('No executable defined')
else:
logging.debug('Running: {}'.format(self.exe))
fargs = [self.exe]
if self.args:
fargs.append(self.args)
fargs = list(flatten(fargs))
loghandle = None
if type(self.logfile) is str:
loghandle = open(self.logfile, 'w')
elif type(self.logfile) is file:
loghandle = self.logfile
elif self.logfile is not None:
raise TypeError('logfile must be a filename, filehandle, or None')
logging.debug('Going to run {}'.format(fargs))
returnval = subprocess.call(fargs, stdout=loghandle, stderr=loghandle)
if returnval != 0:
raise SimulationError('return value not zero')
if type(self.logfile) is str:
loghandle.close()
def run(self):
"""Run the simulation"""
if self.exe is None:
raise SimulationError('No executable defined')
else:
logging.debug('Running: {}'.format(self.exe))
fargs = [self.exe]
if self.args:
fargs.append(self.args)
fargs = list(flatten(fargs))
loghandle = None
if type(self.logfile) is str:
loghandle = open(self.logfile, 'w')
elif type(self.logfile) is file:
loghandle = self.logfile
elif self.logfile is not None:
raise TypeError('logfile must be a filename, filehandle, or None')
logging.debug('Going to run {}'.format(fargs))
returnval = subprocess.call(fargs, stdout=loghandle, stderr=loghandle)
if returnval != 0:
raise SimulationError('return value not zero')
if type(self.logfile) is str:
loghandle.close()
inputs,
sequence_length=None,
initial_state_fw=None,
initial_state_bw=None,
dtype=None,
parallel_iterations=None,
swap_memory=False,
time_major=False,
scope=None):
assert not time_major
flat_inputs = flatten(inputs, 2) # [-1, J, d]
flat_len = None if sequence_length is None else tf.cast(
flatten(sequence_length, 0), 'int64')
(flat_fw_outputs, flat_bw_outputs), final_state = \
_bidirectional_dynamic_rnn(cell_fw, cell_bw, flat_inputs, sequence_length=flat_len,
initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw,
dtype=dtype, parallel_iterations=parallel_iterations, swap_memory=swap_memory,
time_major=time_major, scope=scope)
fw_outputs = reconstruct(flat_fw_outputs, inputs, 2)
bw_outputs = reconstruct(flat_bw_outputs, inputs, 2)
# FIXME : final state is not reshaped!
return (fw_outputs, bw_outputs), final_state
if __name__ == '__main__':
xx = tf.get_variable(name='xx', shape=[15, 22, 200], dtype='float')
jj = flatten(xx, 2)
print(jj)
