def call(self, inputs, states, constants=None, training=None, **kwargs):
# Recover per-cell states.
state_size = (self.state_size[::-1]
if self.reverse_state_order else self.state_size)
nested_states = tf.nest.pack_sequence_as(state_size,
tf.nest.flatten(states))
# Call the cells in order and store the returned states.
new_nested_states = []
for cell, states in zip(self.cells, nested_states):
states = states if tf.nest.is_nested(states) else [states]
# TF cell does not wrap the state into list when there is only one state.
is_tf_rnn_cell = getattr(cell, '_is_tf_rnn_cell', None) is not None
states = states[0] if len(
states) == 1 and is_tf_rnn_cell else states
if generic_utils.has_arg(cell.call, 'training'):
kwargs['training'] = training
else:
kwargs.pop('training', None)
# Use the __call__ function for callable objects, eg layers, so that it
# will have the proper name scopes for the ops, etc.
cell_call_fn = cell.__call__ if callable(cell) else cell.call
if generic_utils.has_arg(cell.call, 'constants'):
inputs, states = cell_call_fn(inputs,
states,
constants=constants,
**kwargs)
else:
inputs, states = cell_call_fn(inputs, states, **kwargs)
new_nested_states.append(states)
return inputs, tf.nest.pack_sequence_as(
state_size, tf.nest.flatten(new_nested_states))
def _call_attend_before(self,
inputs,
cell_states,
attended,
attention_states,
attended_mask,
training=None):
"""Complete attentive cell transformation, if `attend_after=False`.
"""
attention_h, new_attention_states, alignment = self.attention_call(
inputs=inputs,
cell_states=cell_states,
attended=attended,
attention_states=attention_states,
attended_mask=attended_mask,
training=training)
cell_input = self._get_cell_input(inputs, attention_h)
if has_arg(self.cell.call, 'training'):
cell_output, new_cell_states = self.cell.call(cell_input,
cell_states,
training=training)
else:
cell_output, new_cell_states = self.cell.call(
cell_input, cell_states)
output = self._get_output(cell_output, attention_h)
output = concatenate([output, alignment])
return output, new_cell_states + new_attention_states
def _call_attend_after(self,
inputs,
cell_states,
attended,
attention_states,
attended_mask,
training=None):
"""Complete attentive cell transformation, if `attend_after=True`.
"""
attention_h_previous = attention_states[0]
cell_input = self._get_cell_input(inputs, attention_h_previous)
if has_arg(self.cell.call, 'training'):
cell_output, new_cell_states = self.cell.call(cell_input,
cell_states,
training=training)
else:
cell_output, new_cell_states = self.cell.call(
cell_input, cell_states)
attention_h, new_attention_states = self.attention_call(
inputs=inputs,
cell_states=new_cell_states,
attended=attended,
attention_states=attention_states,
attended_mask=attended_mask,
training=training)
output = self._get_output(cell_output, attention_h)
return output, new_cell_states, new_attention_states
def check_params(self, params):
"""Checks for user typos in `params`.
# Arguments
params : dictionary; the parameters to be checked
# Raises
ValueError : if any member of `params` is not a valid argument.
"""
legal_params_fns = [
Sequential.evaluate, Sequential.fit, Sequential.predict,
Sequential.predict_classes, Model.evaluate, Model.fit,
Model.predict
]
if self.build_fn is None:
legal_params_fns.append(self.__call__)
elif (not isinstance(self.build_fn, types.FunctionType)
and not isinstance(self.build_fn, types.MethodType)):
legal_params_fns.append(self.build_fn.__call__)
else:
legal_params_fns.append(self.build_fn)
for params_name in params:
for fn in legal_params_fns:
if has_arg(fn, params_name):
break
else:
if params_name != 'nb_epoch':
raise ValueError(
'{} is not a legal parameter'.format(params_name))
def call(self, inputs, mask=None, training=None, initial_state=None):
# We need to rewrite this `call` method by combining `RNN`'s and `GRU`'s.
self.cell._dropout_mask = None
self.cell._recurrent_dropout_mask = None
self.cell._masking_dropout_mask = None
inputs = inputs[:3]
if initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_state)) + ' initial states.')
timesteps = K.int_shape(inputs[0])[1]
kwargs = {}
if has_arg(self.cell.call, 'training'):
kwargs['training'] = training
def step(inputs, states):
return self.cell.call(inputs, states, **kwargs)
# concatenate the inputs and get the mask
concatenated_inputs = K.concatenate(inputs, axis=-1)
mask = mask[0]
last_output, outputs, states = K.rnn(step,
concatenated_inputs,
initial_state,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=timesteps)
if self.stateful:
updates = []
for i, state in enumerate(states):
updates.append((self.states[i], state))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = last_output
# Properly set learning phase
if getattr(last_output, '_uses_learning_phase', False):
output._uses_learning_phase = True
for state in states:
state._uses_learning_phase = True
if self.return_state:
states = list(states)[:-2] # remove x_keep and ss
return [output] + states
return output
def call(self, inputs, states, constants=None, **kwargs):
nested_states = []
for cell in self.cells[::-1]:
if hasattr(cell.state_size, '__len__'):
nested_states.append(states[:len(cell.state_size)])
states = states[len(cell.state_size):]
else:
nested_states.append([states[0]])
states = states[1:]
nested_states = nested_states[::-1]
new_nested_states = []
for cell, states in zip(self.cells, nested_states):
if has_arg(cell.call, 'constants'):
inputs, states = cell.call(inputs,
states,
constraints=constraints,
**kwargs)
else:
inputs, states = cell.call(inputs, states, **kwargs)
new_nested_states.append(states)
states = []
for cell_states in nested_states[::-1]:
states += cell_states
return inputs, states
def call(self, inputs, training=None, mask=None):
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
ni = inputs.shape[1]
y = self.forward_layer.call(inputs[:, :ni // 2], **kwargs)
y_rev = self.backward_layer.call(inputs[:, ni // 2:], **kwargs)
output = K.concatenate([y, y_rev])
# Properly set learning phase
if self.layer.dropout + self.layer.recurrent_dropout > 0:
output._uses_learning_phase = True
return output
def call(self, inputs, states, constants=None, **kwargs):
# print("ciao")
#reshaped_feature_for_calculation = Reshape((self.feature_shape[0], self.feature_shape[1]))(inputs)
if self.time == 0:
random_state = states[0]
first_map = self.attention_model(random_state)
#first_map = Softmax(first_map)
self.current_map = first_map
self.attention_maps.append(first_map)
self.time += 1
# Recover per-cell states.
#
nested_states = []
for cell in self.cells[::
-1] if self.reverse_state_order else self.cells:
if hasattr(cell.state_size, '__len__'):
nested_states.append(states[:len(cell.state_size)])
states = states[len(cell.state_size):]
else:
nested_states.append([states[0]])
states = states[1:]
if self.reverse_state_order:
nested_states = nested_states[::-1]
# Call the cells in order and store the returned states.
#todo qui bisogna intervenire con la mappa
new_nested_states = []
counter_cells = 0
for cell, states in zip(self.cells, nested_states):
if counter_cells == 0:
inputs = x_calculation([inputs, self.current_map])
if has_arg(cell.call, 'constants'):
inputs, states = cell.call(inputs,
states,
constants=constants,
**kwargs)
else:
inputs, states = cell.call(inputs, states, **kwargs)
new_nested_states.append(states)
if counter_cells == len(self.cells) - 1:
out = self.attention_model(inputs)
self.attention_maps.append(out)
self.current_map = out
counter_cells += 1
# Format the new states as a flat list
# in reverse cell order.
new_states = []
# print(self.current_map)
if self.reverse_state_order:
new_nested_states = new_nested_states[::-1]
for cell_states in new_nested_states:
new_states += cell_states
return inputs, new_states
def call(self,
inputs,
mask=None,
training=None,
initial_state=None,
constants=None):
if isinstance(mask, list):
mask = mask[0]
if initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
kwargs = {}
if generic_utils.has_arg(self.cell.call, 'training'):
kwargs['training'] = training
self.cell._generate_dropout_mask(inputs, training=training)
self.cell._generate_recurrent_dropout_mask(inputs, training=training)
fourier_timesteps = (np.arange(self.cell.state_size[0], dtype=np.float64) + 1) / self.cell.state_size[0]
initial_state = initial_state[0]
# custom K.Rnn for adding fourier_timesteps input, only unroll routine
last_output, outputs, states = self.Adaptive_SFM_Rnn(self.cell.call,
inputs, fourier_timesteps,
initial_state,
constants=constants,
go_backwards=self.go_backwards,
mask=mask)
outputs = tf.squeeze(outputs[:,:,:,-1:], axis=[-1])
last_output = last_output[:,-1:,:]
if self.stateful:
updates = []
for i in range(len(states)):
updates.append((self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = last_output
if getattr(last_output, '_uses_learning_phase', False):
output._uses_learning_phase = True
if self.return_state:
if not isinstance(states, (list, tuple)):
states = [states]
else:
states = list(states)
return [output] + states
else:
return output
def call(self, inputs, training=None, mask=None):
# Copied from https://github.com/fchollet/keras/blob/master/keras/layers/wrappers.py#L100
# (tag 2.1.0) with our modifications marked in comments
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
# Our modification
if has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
# end our modification
uses_learning_phase = False
input_shape = K.int_shape(inputs)
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = _object_list_uid(inputs)
inputs = K.reshape(inputs, (-1, ) + input_shape[2:])
self._input_map[input_uid] = inputs
# (num_samples * timesteps, ...)
# Our modification
if kwargs.get('mask', None) is not None:
mask_shape = K.int_shape(mask)
kwargs['mask'] = K.reshape(kwargs['mask'], (-1, ) + mask_shape[2:])
# end our modification
y = self.layer.call(inputs, **kwargs)
if hasattr(y, '_uses_learning_phase'):
uses_learning_phase = y._uses_learning_phase
# Shape: (num_samples, timesteps, ...)
output_shape = self.compute_output_shape(input_shape)
y = K.reshape(y, (-1, input_length) + output_shape[2:])
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer')
and self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
return y
def test_has_arg(fn, name, accept_all, expected):
if isinstance(fn, str):
context = dict()
try:
exec('def {}: pass'.format(fn), context)
except SyntaxError:
if sys.version_info >= (3,):
raise
pytest.skip('Function is not compatible with Python 2')
context.pop('__builtins__', None) # Sometimes exec adds builtins to the context
fn, = context.values()
assert has_arg(fn, name, accept_all) is expected
def call(self, inputs, training=None, mask=None):
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
uses_learning_phase = False
if not isinstance(inputs, list):
inputs = [inputs]
reshaped_inputs = []
for input in inputs:
input_shape = K.int_shape(input)
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = input_shape[1]
if not input_length:
input_length = K.shape(input)[1]
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = _object_list_uid(input)
input = K.reshape(input, (-1, ) + input_shape[2:])
self._input_map[input_uid] = input
reshaped_inputs.append(input)
# (num_samples * timesteps, ...)
reshaped_initial_states = reshaped_inputs[1:]
reshaped_inputs = reshaped_inputs[0]
if reshaped_initial_states:
kwargs['initial_state'] = reshaped_initial_states
y = self.layer.call(reshaped_inputs, **kwargs)
if hasattr(y, '_uses_learning_phase'):
uses_learning_phase = y._uses_learning_phase
# Shape: (num_samples, timesteps, ...)
input_shape = [K.int_shape(input) for input in inputs]
output_shape = self.compute_output_shape(input_shape)
y = K.reshape(y, (-1, input_length) + output_shape[2:])
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer')
and self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
return y
def __init__(self, params= {}):
self.params =params
#more Keras layers can be added to this dictionary. Visit Keras documentation for more info
self.layers = {'dense': Dense, 'batch_norm': BatchNormalization, 'conv2d': Conv2D, 'flatten': Flatten,
'dropout': Dropout, 'maxpool': MaxPooling2D, 'avgpool'
: AveragePooling2D}
if len(self.params)!=0:
assert isinstance(self.params, dict), print('params parameter passed need to be a dictionary.','fail')
assert all([k in self.params.keys() for k in ['build', 'compile', 'fit']]), print('Params must contain build, compile and fit keys','fail')
assert len(self.params.keys())==3, print('Valid keys for params: build, compile and fit', 'fail')
assert isinstance(self.params['build'],list), print('Value for build key must be a list', 'fail')
assert isinstance(self.params['compile'], dict) and isinstance(self.params['fit'], dict), print('Values for compile and fit keys must be dictionaries', 'fail')
assert all([k for d in self.params['build'] for k, v in d.items() if k in self.layers.keys()]), print('layer type is not valid','fail')
assert all([has_arg(self.layers[k], i) for d in self.params['build'] for k, v in d.items() for i in v.keys() if i not in ['input_dim', 'name','input_shape']]), print('Hyper_parameters for layers are invalid','fail')
assert all([has_arg(Sequential.compile, k) for k in self.params['compile'].keys()]), print('Model compilation parameters are invalid','fail')
assert all([has_arg(Sequential.fit, k) for k in self.params['fit'].keys()]) , print('Model fit parameters are invalid','fail')
self.model= self.build_model()
else:
print("params dict is empty")
def filter_sk_params(self, fn, override=None):
"""Filters `sk_params` and returns those in `fn`'s arguments.
# Arguments
fn : arbitrary function
override: dictionary, values to override `sk_params`
# Returns
res : dictionary containing variables
in both `sk_params` and `fn`'s arguments.
"""
override = override or {}
res = {}
for name, value in self.sk_params.items():
if has_arg(fn, name):
res.update({name: value})
res.update(override)
return res
def check_params(self, params) -> None:
"""Check for user typos in `params`.
Args:
params: dictionary; the parameters to be checked
Raises:
ValueError: if any member of `params` is not a valid argument.
"""
for params_name in params:
for fn in self._funcs_with_legal_params:
if has_arg(fn, params_name):
break
else:
raise ValueError(f'{params_name} is not a legal parameter')
def check_params(self, params):
"""
Checks for user typos in 'params'.
Arguments:
params: dictionary; the parameters to be checked
Raises:
ValueError: if any member of `params` is not a valid argument.
"""
# Sequential and Models functions reference: https://github.com/keras-team/keras/blob/994c4bb338ce440a27df5db48b8f5044a3e8f611/docs/structure.py
legal_params_fns = [
Sequential.compile,
Sequential.fit,
Sequential.evaluate,
Sequential.predict,
Sequential.train_on_batch,
Sequential.test_on_batch,
Sequential.predict_on_batch,
Sequential.fit_generator,
Sequential.evaluate_generator,
Sequential.predict_generator,
Model.compile,
Model.fit,
Model.evaluate,
Model.predict,
Model.train_on_batch,
Model.test_on_batch,
Model.predict_on_batch,
Model.fit_generator,
Model.evaluate_generator,
Model.predict_generator
]
if (not isinstance(self.build_fn, types.FunctionType) and (not isinstance(self.build_fn, types.MethodType))):
raise ValueError('build_fn is not a function or method.')
else:
legal_params_fns.append(self.build_fn)
for params_name in params:
for fn in legal_params_fns:
if has_arg(fn, params_name):
break
else:
raise ValueError('{} is not a legal parameter'.format(params_name))
def check_params(params, fn):
"""
Check whether params are valid for function(s)
Parameter:
----------
params : dict
fn : function or functions iterables
"""
if not isinstance(fn, (list, tuple)):
fn = [fn]
for p in list(six.iterkeys(params)):
for f in fn:
if has_arg(f, p):
break
else:
raise ValueError(
"{} is not a legal parameter".format(p))
def _filter_hyperparams(self, fn, override=None):
"""Filter `hyperparams` and return those in `fn`'s arguments.
Args:
fn : arbitrary function
override: dictionary, values to override `hyperparams`
Returns:
res : dictionary containing variables
in both `hyperparams` and `fn`'s arguments.
"""
override = override or {}
res = {}
for name, value in self.hyperparams.items():
if has_arg(fn, name):
res.update({name: value})
res.update(override)
return res
def check_params(self, params):
"""Check for user typos in `params`.
# Arguments
params: dictionary; the parameters to be checked
# Raises
ValueError: if any member of `params` is not a valid argument.
"""
legal_params_fns = [
Sequential.fit, Sequential.fit_generator, Sequential.predict,
Sequential.predict_classes, Sequential.evaluate, self.__model__,
self.__callbacks__
]
for params_name in params:
for fn in legal_params_fns:
if has_arg(fn, params_name):
break
else:
raise ValueError(f'{params_name} is not a legal parameter')
def _filter_n_set_params(self, fn, **params):
"""
Filter `params` and return fn(args).
Arguments
----------
fn: function
function to retrun.
**params: dictonary.
Dictionary of parameter names mapped to their values.
Returns
----------
fn(expected args)
"""
args = {}
for params_name in params:
if has_arg(fn, params_name):
args[params_name] = params[params_name]
return fn(**args)
def _check_params(self, params, legal_params_fns):
"""
Checks for user typos in "params" for common functions.
Arguments
----------
params: dictionary.
the parameters to be checked.
legal_params_fns: list of functions.
the functions to be checked
Raises
----------
ValueError: if any member of `params` is not a valid argument.
"""
for params_name in params:
for fn in legal_params_fns:
if has_arg(fn, params_name):
break
else:
raise ValueError('{} is not a legal parameter'.format(params_name))
def call(self, inputs, training=None):
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
uses_learning_phase = False
input_shape = K.int_shape(inputs)
if False and input_shape[0]:
# batch size matters, use rnn-based implementation
def step(x, _):
global uses_learning_phase
output = self.layer.call(x, **kwargs)
if hasattr(output, '_uses_learning_phase'):
uses_learning_phase = (output._uses_learning_phase
or uses_learning_phase)
return output, []
_, outputs, _ = K.rnn(step,
inputs,
initial_states=[],
input_length=input_shape[1],
unroll=False)
y = outputs
else:
'''
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = _object_list_uid(inputs)
inputs = K.reshape(inputs, (-1,) + input_shape[2:])
self._input_map[input_uid] = inputs
# (num_samples * timesteps, ...)
y = self.layer.call(inputs, **kwargs)
if hasattr(y, '_uses_learning_phase'):
uses_learning_phase = y._uses_learning_phase
# Shape: (num_samples, timesteps, ...)
output_shape = self.compute_output_shape(input_shape)
y = K.reshape(y, (-1, input_length) + output_shape[2:])
'''
input_length = input_shape[1]
output_shape = self.compute_output_shape(input_shape)
# self.gpuInputVar = tf.zeros( (input_shape[0],) + input_shape[2:])
self.gpuInputVar = None
# with K.device('/cpu:0'):
# self.cpuOutputVar = tf.zeros(output_shape)
# tsteps = None
# with K.device('/cpu:0'):
# tsteps = K.unstack(inputs, axis=1)
# outList = []
# ins = K.reshape(inputs, (input_length, input_shape[0]) + input_shape[2:])
numDims = len(input_shape)
with K.device('/cpu:0'):
ins = tf.transpose(inputs, (1, 0) +
tuple([i for i in range(2, numDims)]))
def foo(timestepPlusDim):
self.gpuInputVar = timestepPlusDim
# with tf.device('/gpu:0'):
# self.gpuInputVar = tf.squeeze(timestepPlusDim)
# with tf.device('/cpu:0'):
# return self.layer.call(self.gpuInputVar)
with tf.device('/gpu:0'):
out = self.layer.call(self.gpuInputVar)
with tf.device('/cpu:0'):
return out
with tf.device('/cpu:0'):
y = tf.map_fn(foo, ins, back_prop=True, swap_memory=True)
'''
# this should be on the device we are otherwise using... gpu...
# for i, timestep in enumerate(tsteps):
for timestep in tsteps:
# out = None
# with tf.device('/gpu:0'):
# test = tsteps[i]
# with tf.device('/cpu:0'):
# gpuVar = tf.identity(tsteps[i])
gpuVar = timestep * 1
# with K.device('/cpu:0'):
# out = self.layer.call(gpuVar, **kwargs)
out = self.layer.call(gpuVar, **kwargs)
with tf.device('/cpu:0'):
cpuVar = out * 1
# outList.append(self.layer.call(tsteps[i] , **kwargs))
outList.append(cpuVar)
# tsteps[i] = K.reshape(tsteps[i], (-1, 1) + output_shape[2:])
# '''
# ins = K.reshape(inputs, (-1,) + input_shape[2:])
# ins = tf.transpose(inputs, (1, 0,2,3))
# def test(timestep):
# with tf.device('/cpu:0'):
# return self.layer.call(timestep)
# y = tf.map_fn(test, ins)
# print(len(outList))
# with tf.device('/cpu:0'):
# with K.device('/cpu:0'):
# y = K.stack(outList, axis=1)
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer')
and self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
return y
# '''
def call(self,
inputs,
mask=None,
training=None,
initial_state=None,
constants=None):
# The input should be dense, padded with zeros. If a ragged input is fed
# into the layer, it is padded and the row lengths are used for masking.
inputs, row_lengths = backend.convert_inputs_if_ragged(inputs)
is_ragged_input = (row_lengths is not None)
self._validate_args_if_ragged(is_ragged_input, mask)
inputs, initial_state, constants = self._process_inputs(
inputs, initial_state, constants)
self._maybe_reset_cell_dropout_mask(self.cell)
if isinstance(self.cell, StackedRNNCells):
for cell in self.cell.cells:
self._maybe_reset_cell_dropout_mask(cell)
if mask is not None:
# Time step masks must be the same for each input.
# TODO(scottzhu): Should we accept multiple different masks?
mask = tf.nest.flatten(mask)[0]
if tf.nest.is_nested(inputs):
# In the case of nested input, use the first element for shape check.
input_shape = backend.int_shape(tf.nest.flatten(inputs)[0])
else:
input_shape = backend.int_shape(inputs)
timesteps = input_shape[0] if self.time_major else input_shape[1]
if self.unroll and timesteps is None:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined. \n'
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer. If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.\n'
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
kwargs = {}
if generic_utils.has_arg(self.cell.call, 'training'):
kwargs['training'] = training
# TF RNN cells expect single tensor as state instead of list wrapped tensor.
is_tf_rnn_cell = getattr(self.cell, '_is_tf_rnn_cell',
None) is not None
# Use the __call__ function for callable objects, eg layers, so that it
# will have the proper name scopes for the ops, etc.
cell_call_fn = self.cell.__call__ if callable(
self.cell) else self.cell.call
if constants:
if not generic_utils.has_arg(self.cell.call, 'constants'):
raise ValueError(
f'RNN cell {self.cell} does not support constants. '
f'Received: constants={constants}')
def step(inputs, states):
constants = states[-self._num_constants:] # pylint: disable=invalid-unary-operand-type
states = states[:-self._num_constants] # pylint: disable=invalid-unary-operand-type
states = states[0] if len(
states) == 1 and is_tf_rnn_cell else states
output, new_states = cell_call_fn(inputs,
states,
constants=constants,
**kwargs)
if not tf.nest.is_nested(new_states):
new_states = [new_states]
return output, new_states
else:
def step(inputs, states):
states = states[0] if len(
states) == 1 and is_tf_rnn_cell else states
output, new_states = cell_call_fn(inputs, states, **kwargs)
if not tf.nest.is_nested(new_states):
new_states = [new_states]
return output, new_states
last_output, outputs, states = backend.rnn(
step,
inputs,
initial_state,
constants=constants,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=row_lengths if row_lengths is not None else timesteps,
time_major=self.time_major,
zero_output_for_mask=self.zero_output_for_mask)
if self.stateful:
updates = [
tf.compat.v1.assign(self_state,
tf.cast(state, self_state.dtype))
for self_state, state in zip(tf.nest.flatten(self.states),
tf.nest.flatten(states))
]
self.add_update(updates)
if self.return_sequences:
output = backend.maybe_convert_to_ragged(
is_ragged_input,
outputs,
row_lengths,
go_backwards=self.go_backwards)
else:
output = last_output
if self.return_state:
if not isinstance(states, (list, tuple)):
states = [states]
else:
states = list(states)
return generic_utils.to_list(output) + states
else:
return output
def call(self,
inputs,
mask=None,
training=None,
initial_state=None,
constants=None):
# note that the .build() method of subclasses MUST define
# self.input_spec and self.state_spec with complete input shapes.
if isinstance(inputs, list):
inputs = inputs[0]
if initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if isinstance(mask, list):
mask = mask[0]
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_state)) + ' initial states.')
timesteps = K.int_shape(inputs)[1]
kwargs = {}
if has_arg(self.cell.call, 'training'):
kwargs['training'] = training
if constants:
if not has_arg(self.cell.call, 'constants'):
raise ValueError('RNN cell does not support constants')
def step(inputs, states):
constants = states[-self._num_constants:]
states = states[:-self._num_constants]
return self.cell.call(inputs,
states,
constants=constants,
**kwargs)
else:
def step(inputs, states):
return self.cell.call(inputs, states, **kwargs)
last_output, outputs, states = K.rnn(step,
inputs,
initial_state,
constants=constants,
go_backwards=self.go_backwards,
mask=mask,
input_length=timesteps)
if self.stateful:
updates = []
for i in range(len(states)):
updates.append((self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = last_output
# Properly set learning phase
if getattr(last_output, '_uses_learning_phase', False):
output._uses_learning_phase = True
if self.return_state:
states = to_list(states, allow_tuple=True)
return [output] + states
else:
return output
def call(self, inputs, mask=None, training=None, initial_state=None):
# input shape: `(samples, time (padded with zeros), input_dim)`
# note that the .build() method of subclasses MUST define
# self.input_spec and self.state_spec with complete input shapes.
if isinstance(inputs, list):
initial_state = inputs[1:]
inputs = inputs[0]
elif initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if isinstance(mask, list):
mask = mask[0]
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_state)) + ' initial states.')
input_shape = K.int_shape(inputs)
print("input:{}".format(input_shape[1]))
timesteps = input_shape[1]
if self.unroll and timesteps in [None, 1]:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined or equal to 1. \n'
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer. If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.\n'
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
if has_arg(self.cell.call, 'training'):
step = functools.partial(self.cell.call, training=training)
else:
step = self.cell.call
'''
last_output, outputs, states, sT = K_new.rnn(step,
inputs,
initial_state,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=timesteps)
'''
last_output, outputs, states = K.rnn(step,
inputs,
initial_state,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=timesteps)
print("OUTPUTS: {}, {}, {}".format(K.int_shape(last_output),
K.int_shape(outputs), states))
if self.stateful:
updates = []
for i in range(len(states)):
updates.append((self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = last_output
# Properly set learning phase
if getattr(last_output, '_uses_learning_phase', False):
output._uses_learning_phase = True
if self.return_state:
if not isinstance(states, (list, tuple)):
states = [states]
else:
states = list(states)
return [output] + list(states)
else:
return output
def call(
self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None,
):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, "training"):
kwargs["training"] = training
if generic_utils.has_arg(self.layer.call, "mask"):
kwargs["mask"] = mask
if generic_utils.has_arg(self.layer.call, "constants"):
kwargs["constants"] = constants
if generic_utils.has_arg(self.layer.call, "initial_state"):
if isinstance(inputs, list) and len(inputs) > 1:
# initial_states are keras tensors, which means they are passed
# in together with inputs as list. The initial_states need to be
# split into forward and backward section, and be feed to layers
# accordingly.
forward_inputs = [inputs[0]]
backward_inputs = [inputs[0]]
pivot = (len(inputs) - self._num_constants) // 2 + 1
# add forward initial state
forward_inputs += inputs[1:pivot]
if not self._num_constants:
# add backward initial state
backward_inputs += inputs[pivot:]
else:
# add backward initial state
backward_inputs += inputs[pivot : -self._num_constants]
# add constants for forward and backward layers
forward_inputs += inputs[-self._num_constants :]
backward_inputs += inputs[-self._num_constants :]
forward_state, backward_state = None, None
if "constants" in kwargs:
kwargs["constants"] = None
elif initial_state is not None:
# initial_states are not keras tensors, eg eager tensor from np
# array. They are only passed in from kwarg initial_state, and
# should be passed to forward/backward layer via kwarg
# initial_state as well.
forward_inputs, backward_inputs = inputs, inputs
half = len(initial_state) // 2
forward_state = initial_state[:half]
backward_state = initial_state[half:]
else:
forward_inputs, backward_inputs = inputs, inputs
forward_state, backward_state = None, None
y = self.forward_layer(
forward_inputs, initial_state=forward_state, **kwargs
)
y_rev = self.backward_layer(
backward_inputs, initial_state=backward_state, **kwargs
)
else:
y = self.forward_layer(inputs, **kwargs)
y_rev = self.backward_layer(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
time_dim = (
0 if getattr(self.forward_layer, "time_major", False) else 1
)
y_rev = backend.reverse(y_rev, time_dim)
if self.merge_mode == "concat":
output = backend.concatenate([y, y_rev])
elif self.merge_mode == "sum":
output = y + y_rev
elif self.merge_mode == "ave":
output = (y + y_rev) / 2
elif self.merge_mode == "mul":
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError(
"Unrecognized value for `merge_mode`. "
f"Received: {self.merge_mode}"
'Expected values are ["concat", "sum", "ave", "mul"]'
)
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def test_has_arg_positional_only():
assert has_arg(pow, 'x') is False
def call(self,
inputs,
mask=None,
training=None,
initial_state=None,
constants=None):
# input shape: `(samples, time (padded with zeros), input_dim)`
# note that the .build() method of subclasses MUST define
# self.input_spec and self.state_spec with complete input shapes.
if isinstance(inputs, list):
inputs = inputs[0]
if initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if isinstance(mask, list):
mask = mask[0]
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_state)) +
' initial states.')
input_shape = K.int_shape(inputs)
timesteps = input_shape[1]
if self.unroll and timesteps in [None, 1]:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined or equal to 1. \n'
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer. If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.\n'
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
kwargs = {}
if has_arg(self.cell.call, 'training'):
kwargs['training'] = training
#print(constants)
#print(initial_state, 'in call function')
#if constants:
if constants is not None:
if not has_arg(self.cell.call, 'constants'):
raise ValueError('RNN cell does not support constants')
if isinstance(constants, (list, tuple)):
self._num_constants = len(constants)
else:
self._num_constants = 1
def step(inputs, states):
#print(states,'states in step')
#print(self._num_constants, '~~~')
constants = states[-self._num_constants:]
states = states[:-self._num_constants]
return self.cell.call(inputs, states, constants=constants,
**kwargs)
else:
def step(inputs, states):
return self.cell.call(inputs, states, **kwargs)
last_output, outputs, states = K.rnn(step,
inputs,
initial_state,
constants=constants,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=timesteps)
if self.stateful:
updates = []
for i in range(len(states)):
updates.append((self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = last_output
# Properly set learning phase
if getattr(last_output, '_uses_learning_phase', False):
output._uses_learning_phase = True
for state in states:
state._uses_learning_phase = True
if self.return_state:
if not isinstance(states, (list, tuple)):
states = [states]
else:
states = list(states)
return [output] + states
else:
return output
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if generic_utils.has_arg(self.layer.call, 'initial_state'):
if isinstance(inputs, list) and len(inputs) > 1:
# initial_states are keras tensors, which means they are passed in
# together with inputs as list. The initial_states need to be split into
# forward and backward section, and be feed to layers accordingly.
forward_inputs = [inputs[0]]
backward_inputs = [inputs[0]]
pivot = (len(inputs) - self._num_constants) // 2 + 1
# add forward initial state
forward_inputs += inputs[1:pivot]
if not self._num_constants:
# add backward initial state
backward_inputs += inputs[pivot:]
else:
# add backward initial state
backward_inputs += inputs[pivot:-self._num_constants]
# add constants for forward and backward layers
forward_inputs += inputs[-self._num_constants:]
backward_inputs += inputs[-self._num_constants:]
forward_state, backward_state = None, None
if 'constants' in kwargs:
kwargs['constants'] = None
elif initial_state is not None:
# initial_states are not keras tensors, eg eager tensor from np array.
# They are only passed in from kwarg initial_state, and should be passed
# to forward/backward layer via kwarg initial_state as well.
forward_inputs, backward_inputs = inputs, inputs
half = len(initial_state) // 2
forward_state = initial_state[:half]
backward_state = initial_state[half:]
else:
forward_inputs, backward_inputs = inputs, inputs
forward_state, backward_state = None, None
y = self.forward_layer(forward_inputs,
initial_state=forward_state, **kwargs)
y_rev = self.backward_layer(backward_inputs,
initial_state=backward_state, **kwargs)
else:
y = self.forward_layer(inputs, **kwargs)
y_rev = self.backward_layer(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
time_dim = 0 if getattr(self.forward_layer, 'time_major', False) else 1
y_rev = K.reverse(y_rev, time_dim)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError(
'Unrecognized value for `merge_mode`: %s' % (self.merge_mode))
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def call(self, inputs, training=None, mask=None):
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
input_shape = tf.nest.map_structure(
lambda x: tf.TensorShape(K.int_shape(x)), inputs)
batch_size = tf_utils.convert_shapes(input_shape)
batch_size = tf.nest.flatten(batch_size)[0]
if batch_size and not self._always_use_reshape:
inputs, row_lengths = K.convert_inputs_if_ragged(inputs)
is_ragged_input = row_lengths is not None
input_length = tf_utils.convert_shapes(input_shape)
input_length = tf.nest.flatten(input_length)[1]
# batch size matters, use rnn-based implementation
def step(x, _):
output = self.layer(x, **kwargs)
return output, []
_, outputs, _ = K.rnn(
step,
inputs,
initial_states=[],
input_length=row_lengths[0] if is_ragged_input else input_length,
mask=mask,
unroll=False)
# pylint: disable=g-long-lambda
y = tf.nest.map_structure(
lambda output: K.maybe_convert_to_ragged(is_ragged_input, output,
row_lengths), outputs)
else:
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
is_ragged_input = tf.nest.map_structure(
lambda x: isinstance(x, tf.RaggedTensor), inputs)
is_ragged_input = tf.nest.flatten(is_ragged_input)
if all(is_ragged_input):
input_values = tf.nest.map_structure(lambda x: x.values, inputs)
input_row_lenghts = tf.nest.map_structure(
lambda x: x.nested_row_lengths()[0], inputs)
y = self.layer(input_values, **kwargs)
y = tf.nest.map_structure(tf.RaggedTensor.from_row_lengths, y,
input_row_lenghts)
elif any(is_ragged_input):
raise ValueError('All inputs has to be either ragged or not, '
'but not mixed. You passed: {}'.format(inputs))
else:
input_length = tf_utils.convert_shapes(input_shape)
input_length = tf.nest.flatten(input_length)[1]
if not input_length:
input_length = tf.nest.map_structure(lambda x: tf.compat.v1.shape(x)[1],
inputs)
input_length = generic_utils.to_list(tf.nest.flatten(input_length))[0]
inner_input_shape = tf.nest.map_structure(
lambda x: self._get_shape_tuple((-1,), x, 2), inputs)
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
inputs = tf.__internal__.nest.map_structure_up_to(inputs, tf.reshape, inputs,
inner_input_shape)
# (num_samples * timesteps, ...)
if generic_utils.has_arg(self.layer.call, 'mask') and mask is not None:
inner_mask_shape = self._get_shape_tuple((-1,), mask, 2)
kwargs['mask'] = K.reshape(mask, inner_mask_shape)
y = self.layer(inputs, **kwargs)
# Shape: (num_samples, timesteps, ...)
output_shape = self.compute_output_shape(input_shape)
# pylint: disable=g-long-lambda
output_shape = tf.nest.map_structure(
lambda tensor, int_shape: self._get_shape_tuple(
(-1, input_length), tensor, 1, int_shape[2:]), y, output_shape)
y = tf.__internal__.nest.map_structure_up_to(y, tf.reshape, y, output_shape)
if not tf.executing_eagerly():
# Set the static shape for the result since it might be lost during
# array_ops reshape, eg, some `None` dim in the result could be
# inferred.
tf.__internal__.nest.map_structure_up_to(
y, lambda tensor, shape: tensor.set_shape(shape), y,
self.compute_output_shape(input_shape))
return y
def call(self, inputs, training=None, mask=None):
if not isinstance(inputs, list):
return super(TimeDistributedMultiInput, self).call(inputs,
training=training,
mask=mask)
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
uses_learning_phase = False
input_shapes = [K.int_shape(inp) for inp in inputs]
batch_sizes = [shape[0] for shape in input_shapes if shape is not None]
fixed_batch_size = any([bs is not None for bs in batch_sizes])
if fixed_batch_size:
# batch size matters, use rnn-based implementation
def step(x, _):
global uses_learning_phase
output = self.layer.call(x, **kwargs)
if hasattr(output, '_uses_learning_phase'):
uses_learning_phase = (output._uses_learning_phase or
uses_learning_phase)
return output, []
# Note: will likely fail here if K.rnn doesn't like multiple inputs
_, outputs, _ = K.rnn(step, inputs,
initial_states=[],
input_length=input_shapes[1],
unroll=False)
y = outputs
else:
# No batch size specified, therefore the layer will be able
# to process batches of any size.
# We can go with reshape-based implementation for performance.
input_length = self.timesteps if self.timesteps else K.shape(inputs[0])[1]
# ^^ assumes input 0 has the correct number of timesteps
def prep_input(inp):
inner_input_shape = self._get_shape_tuple((-1,), inp, 2)
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = object_list_uid(inp)
reshaped = K.reshape(inp, inner_input_shape)
self._input_map[input_uid] = reshaped
return reshaped
inputs = [prep_input(inp) for inp in inputs]
# (num_samples * timesteps, ...)
if has_arg(self.layer.call, 'mask') and mask is not None:
inner_mask_shape = self._get_shape_tuple((-1,), mask, 2)
kwargs['mask'] = K.reshape(mask, inner_mask_shape)
y = self.layer.call(inputs, **kwargs)
if isinstance(y, list):
raise NotImplementedError(
'TimeDistributedMultiInput not implemented for multiple '
'output tensors yet.')
if hasattr(y, '_uses_learning_phase'):
uses_learning_phase = y._uses_learning_phase
# Shape: (num_samples, timesteps, ...)
output_shape = self.compute_output_shape(input_shapes)
output_shape = self._get_shape_tuple(
(-1, input_length), y, 1, output_shape[2:])
y = K.reshape(y, output_shape)
# Apply activity regularizer if any:
if (hasattr(self.layer, 'activity_regularizer') and
self.layer.activity_regularizer is not None):
regularization_loss = self.layer.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
return y
