def call(self, inputs, states, constants=None, training=None, **kwargs): # Recover per-cell states. state_size = (self.state_size[::-1] if self.reverse_state_order else self.state_size) nested_states = tf.nest.pack_sequence_as(state_size, tf.nest.flatten(states)) # Call the cells in order and store the returned states. new_nested_states = [] for cell, states in zip(self.cells, nested_states): states = states if tf.nest.is_nested(states) else [states] # TF cell does not wrap the state into list when there is only one state. is_tf_rnn_cell = getattr(cell, '_is_tf_rnn_cell', None) is not None states = states[0] if len( states) == 1 and is_tf_rnn_cell else states if generic_utils.has_arg(cell.call, 'training'): kwargs['training'] = training else: kwargs.pop('training', None) # Use the __call__ function for callable objects, eg layers, so that it # will have the proper name scopes for the ops, etc. cell_call_fn = cell.__call__ if callable(cell) else cell.call if generic_utils.has_arg(cell.call, 'constants'): inputs, states = cell_call_fn(inputs, states, constants=constants, **kwargs) else: inputs, states = cell_call_fn(inputs, states, **kwargs) new_nested_states.append(states) return inputs, tf.nest.pack_sequence_as( state_size, tf.nest.flatten(new_nested_states))
def _call_attend_before(self, inputs, cell_states, attended, attention_states, attended_mask, training=None): """Complete attentive cell transformation, if `attend_after=False`. """ attention_h, new_attention_states, alignment = self.attention_call( inputs=inputs, cell_states=cell_states, attended=attended, attention_states=attention_states, attended_mask=attended_mask, training=training) cell_input = self._get_cell_input(inputs, attention_h) if has_arg(self.cell.call, 'training'): cell_output, new_cell_states = self.cell.call(cell_input, cell_states, training=training) else: cell_output, new_cell_states = self.cell.call( cell_input, cell_states) output = self._get_output(cell_output, attention_h) output = concatenate([output, alignment]) return output, new_cell_states + new_attention_states
def _call_attend_after(self, inputs, cell_states, attended, attention_states, attended_mask, training=None): """Complete attentive cell transformation, if `attend_after=True`. """ attention_h_previous = attention_states[0] cell_input = self._get_cell_input(inputs, attention_h_previous) if has_arg(self.cell.call, 'training'): cell_output, new_cell_states = self.cell.call(cell_input, cell_states, training=training) else: cell_output, new_cell_states = self.cell.call( cell_input, cell_states) attention_h, new_attention_states = self.attention_call( inputs=inputs, cell_states=new_cell_states, attended=attended, attention_states=attention_states, attended_mask=attended_mask, training=training) output = self._get_output(cell_output, attention_h) return output, new_cell_states, new_attention_states
def check_params(self, params): """Checks for user typos in `params`. # Arguments params : dictionary; the parameters to be checked # Raises ValueError : if any member of `params` is not a valid argument. """ legal_params_fns = [ Sequential.evaluate, Sequential.fit, Sequential.predict, Sequential.predict_classes, Model.evaluate, Model.fit, Model.predict ] if self.build_fn is None: legal_params_fns.append(self.__call__) elif (not isinstance(self.build_fn, types.FunctionType) and not isinstance(self.build_fn, types.MethodType)): legal_params_fns.append(self.build_fn.__call__) else: legal_params_fns.append(self.build_fn) for params_name in params: for fn in legal_params_fns: if has_arg(fn, params_name): break else: if params_name != 'nb_epoch': raise ValueError( '{} is not a legal parameter'.format(params_name))
def call(self, inputs, mask=None, training=None, initial_state=None): # We need to rewrite this `call` method by combining `RNN`'s and `GRU`'s. self.cell._dropout_mask = None self.cell._recurrent_dropout_mask = None self.cell._masking_dropout_mask = None inputs = inputs[:3] if initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') timesteps = K.int_shape(inputs[0])[1] kwargs = {} if has_arg(self.cell.call, 'training'): kwargs['training'] = training def step(inputs, states): return self.cell.call(inputs, states, **kwargs) # concatenate the inputs and get the mask concatenated_inputs = K.concatenate(inputs, axis=-1) mask = mask[0] last_output, outputs, states = K.rnn(step, concatenated_inputs, initial_state, go_backwards=self.go_backwards, mask=mask, unroll=self.unroll, input_length=timesteps) if self.stateful: updates = [] for i, state in enumerate(states): updates.append((self.states[i], state)) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = last_output # Properly set learning phase if getattr(last_output, '_uses_learning_phase', False): output._uses_learning_phase = True for state in states: state._uses_learning_phase = True if self.return_state: states = list(states)[:-2] # remove x_keep and ss return [output] + states return output
def call(self, inputs, states, constants=None, **kwargs): nested_states = [] for cell in self.cells[::-1]: if hasattr(cell.state_size, '__len__'): nested_states.append(states[:len(cell.state_size)]) states = states[len(cell.state_size):] else: nested_states.append([states[0]]) states = states[1:] nested_states = nested_states[::-1] new_nested_states = [] for cell, states in zip(self.cells, nested_states): if has_arg(cell.call, 'constants'): inputs, states = cell.call(inputs, states, constraints=constraints, **kwargs) else: inputs, states = cell.call(inputs, states, **kwargs) new_nested_states.append(states) states = [] for cell_states in nested_states[::-1]: states += cell_states return inputs, states
def call(self, inputs, training=None, mask=None): kwargs = {} if has_arg(self.layer.call, 'training'): kwargs['training'] = training if has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask ni = inputs.shape[1] y = self.forward_layer.call(inputs[:, :ni // 2], **kwargs) y_rev = self.backward_layer.call(inputs[:, ni // 2:], **kwargs) output = K.concatenate([y, y_rev]) # Properly set learning phase if self.layer.dropout + self.layer.recurrent_dropout > 0: output._uses_learning_phase = True return output
def call(self, inputs, states, constants=None, **kwargs): # print("ciao") #reshaped_feature_for_calculation = Reshape((self.feature_shape[0], self.feature_shape[1]))(inputs) if self.time == 0: random_state = states[0] first_map = self.attention_model(random_state) #first_map = Softmax(first_map) self.current_map = first_map self.attention_maps.append(first_map) self.time += 1 # Recover per-cell states. # nested_states = [] for cell in self.cells[:: -1] if self.reverse_state_order else self.cells: if hasattr(cell.state_size, '__len__'): nested_states.append(states[:len(cell.state_size)]) states = states[len(cell.state_size):] else: nested_states.append([states[0]]) states = states[1:] if self.reverse_state_order: nested_states = nested_states[::-1] # Call the cells in order and store the returned states. #todo qui bisogna intervenire con la mappa new_nested_states = [] counter_cells = 0 for cell, states in zip(self.cells, nested_states): if counter_cells == 0: inputs = x_calculation([inputs, self.current_map]) if has_arg(cell.call, 'constants'): inputs, states = cell.call(inputs, states, constants=constants, **kwargs) else: inputs, states = cell.call(inputs, states, **kwargs) new_nested_states.append(states) if counter_cells == len(self.cells) - 1: out = self.attention_model(inputs) self.attention_maps.append(out) self.current_map = out counter_cells += 1 # Format the new states as a flat list # in reverse cell order. new_states = [] # print(self.current_map) if self.reverse_state_order: new_nested_states = new_nested_states[::-1] for cell_states in new_nested_states: new_states += cell_states return inputs, new_states
def call(self, inputs, mask=None, training=None, initial_state=None, constants=None): if isinstance(mask, list): mask = mask[0] if initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) kwargs = {} if generic_utils.has_arg(self.cell.call, 'training'): kwargs['training'] = training self.cell._generate_dropout_mask(inputs, training=training) self.cell._generate_recurrent_dropout_mask(inputs, training=training) fourier_timesteps = (np.arange(self.cell.state_size[0], dtype=np.float64) + 1) / self.cell.state_size[0] initial_state = initial_state[0] # custom K.Rnn for adding fourier_timesteps input, only unroll routine last_output, outputs, states = self.Adaptive_SFM_Rnn(self.cell.call, inputs, fourier_timesteps, initial_state, constants=constants, go_backwards=self.go_backwards, mask=mask) outputs = tf.squeeze(outputs[:,:,:,-1:], axis=[-1]) last_output = last_output[:,-1:,:] if self.stateful: updates = [] for i in range(len(states)): updates.append((self.states[i], states[i])) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = last_output if getattr(last_output, '_uses_learning_phase', False): output._uses_learning_phase = True if self.return_state: if not isinstance(states, (list, tuple)): states = [states] else: states = list(states) return [output] + states else: return output
def call(self, inputs, training=None, mask=None): # Copied from https://github.com/fchollet/keras/blob/master/keras/layers/wrappers.py#L100 # (tag 2.1.0) with our modifications marked in comments kwargs = {} if has_arg(self.layer.call, 'training'): kwargs['training'] = training # Our modification if has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask # end our modification uses_learning_phase = False input_shape = K.int_shape(inputs) input_length = input_shape[1] if not input_length: input_length = K.shape(inputs)[1] # Shape: (num_samples * timesteps, ...). And track the # transformation in self._input_map. input_uid = _object_list_uid(inputs) inputs = K.reshape(inputs, (-1, ) + input_shape[2:]) self._input_map[input_uid] = inputs # (num_samples * timesteps, ...) # Our modification if kwargs.get('mask', None) is not None: mask_shape = K.int_shape(mask) kwargs['mask'] = K.reshape(kwargs['mask'], (-1, ) + mask_shape[2:]) # end our modification y = self.layer.call(inputs, **kwargs) if hasattr(y, '_uses_learning_phase'): uses_learning_phase = y._uses_learning_phase # Shape: (num_samples, timesteps, ...) output_shape = self.compute_output_shape(input_shape) y = K.reshape(y, (-1, input_length) + output_shape[2:]) # Apply activity regularizer if any: if (hasattr(self.layer, 'activity_regularizer') and self.layer.activity_regularizer is not None): regularization_loss = self.layer.activity_regularizer(y) self.add_loss(regularization_loss, inputs) if uses_learning_phase: y._uses_learning_phase = True return y
def test_has_arg(fn, name, accept_all, expected): if isinstance(fn, str): context = dict() try: exec('def {}: pass'.format(fn), context) except SyntaxError: if sys.version_info >= (3,): raise pytest.skip('Function is not compatible with Python 2') context.pop('__builtins__', None) # Sometimes exec adds builtins to the context fn, = context.values() assert has_arg(fn, name, accept_all) is expected
def call(self, inputs, training=None, mask=None): kwargs = {} if has_arg(self.layer.call, 'training'): kwargs['training'] = training uses_learning_phase = False if not isinstance(inputs, list): inputs = [inputs] reshaped_inputs = [] for input in inputs: input_shape = K.int_shape(input) # No batch size specified, therefore the layer will be able # to process batches of any size. # We can go with reshape-based implementation for performance. input_length = input_shape[1] if not input_length: input_length = K.shape(input)[1] # Shape: (num_samples * timesteps, ...). And track the # transformation in self._input_map. input_uid = _object_list_uid(input) input = K.reshape(input, (-1, ) + input_shape[2:]) self._input_map[input_uid] = input reshaped_inputs.append(input) # (num_samples * timesteps, ...) reshaped_initial_states = reshaped_inputs[1:] reshaped_inputs = reshaped_inputs[0] if reshaped_initial_states: kwargs['initial_state'] = reshaped_initial_states y = self.layer.call(reshaped_inputs, **kwargs) if hasattr(y, '_uses_learning_phase'): uses_learning_phase = y._uses_learning_phase # Shape: (num_samples, timesteps, ...) input_shape = [K.int_shape(input) for input in inputs] output_shape = self.compute_output_shape(input_shape) y = K.reshape(y, (-1, input_length) + output_shape[2:]) # Apply activity regularizer if any: if (hasattr(self.layer, 'activity_regularizer') and self.layer.activity_regularizer is not None): regularization_loss = self.layer.activity_regularizer(y) self.add_loss(regularization_loss, inputs) if uses_learning_phase: y._uses_learning_phase = True return y
def __init__(self, params= {}): self.params =params #more Keras layers can be added to this dictionary. Visit Keras documentation for more info self.layers = {'dense': Dense, 'batch_norm': BatchNormalization, 'conv2d': Conv2D, 'flatten': Flatten, 'dropout': Dropout, 'maxpool': MaxPooling2D, 'avgpool' : AveragePooling2D} if len(self.params)!=0: assert isinstance(self.params, dict), print('params parameter passed need to be a dictionary.','fail') assert all([k in self.params.keys() for k in ['build', 'compile', 'fit']]), print('Params must contain build, compile and fit keys','fail') assert len(self.params.keys())==3, print('Valid keys for params: build, compile and fit', 'fail') assert isinstance(self.params['build'],list), print('Value for build key must be a list', 'fail') assert isinstance(self.params['compile'], dict) and isinstance(self.params['fit'], dict), print('Values for compile and fit keys must be dictionaries', 'fail') assert all([k for d in self.params['build'] for k, v in d.items() if k in self.layers.keys()]), print('layer type is not valid','fail') assert all([has_arg(self.layers[k], i) for d in self.params['build'] for k, v in d.items() for i in v.keys() if i not in ['input_dim', 'name','input_shape']]), print('Hyper_parameters for layers are invalid','fail') assert all([has_arg(Sequential.compile, k) for k in self.params['compile'].keys()]), print('Model compilation parameters are invalid','fail') assert all([has_arg(Sequential.fit, k) for k in self.params['fit'].keys()]) , print('Model fit parameters are invalid','fail') self.model= self.build_model() else: print("params dict is empty")
def filter_sk_params(self, fn, override=None): """Filters `sk_params` and returns those in `fn`'s arguments. # Arguments fn : arbitrary function override: dictionary, values to override `sk_params` # Returns res : dictionary containing variables in both `sk_params` and `fn`'s arguments. """ override = override or {} res = {} for name, value in self.sk_params.items(): if has_arg(fn, name): res.update({name: value}) res.update(override) return res
def check_params(self, params) -> None: """Check for user typos in `params`. Args: params: dictionary; the parameters to be checked Raises: ValueError: if any member of `params` is not a valid argument. """ for params_name in params: for fn in self._funcs_with_legal_params: if has_arg(fn, params_name): break else: raise ValueError(f'{params_name} is not a legal parameter')
def check_params(self, params): """ Checks for user typos in 'params'. Arguments: params: dictionary; the parameters to be checked Raises: ValueError: if any member of `params` is not a valid argument. """ # Sequential and Models functions reference: https://github.com/keras-team/keras/blob/994c4bb338ce440a27df5db48b8f5044a3e8f611/docs/structure.py legal_params_fns = [ Sequential.compile, Sequential.fit, Sequential.evaluate, Sequential.predict, Sequential.train_on_batch, Sequential.test_on_batch, Sequential.predict_on_batch, Sequential.fit_generator, Sequential.evaluate_generator, Sequential.predict_generator, Model.compile, Model.fit, Model.evaluate, Model.predict, Model.train_on_batch, Model.test_on_batch, Model.predict_on_batch, Model.fit_generator, Model.evaluate_generator, Model.predict_generator ] if (not isinstance(self.build_fn, types.FunctionType) and (not isinstance(self.build_fn, types.MethodType))): raise ValueError('build_fn is not a function or method.') else: legal_params_fns.append(self.build_fn) for params_name in params: for fn in legal_params_fns: if has_arg(fn, params_name): break else: raise ValueError('{} is not a legal parameter'.format(params_name))
def check_params(params, fn): """ Check whether params are valid for function(s) Parameter: ---------- params : dict fn : function or functions iterables """ if not isinstance(fn, (list, tuple)): fn = [fn] for p in list(six.iterkeys(params)): for f in fn: if has_arg(f, p): break else: raise ValueError( "{} is not a legal parameter".format(p))
def _filter_hyperparams(self, fn, override=None): """Filter `hyperparams` and return those in `fn`'s arguments. Args: fn : arbitrary function override: dictionary, values to override `hyperparams` Returns: res : dictionary containing variables in both `hyperparams` and `fn`'s arguments. """ override = override or {} res = {} for name, value in self.hyperparams.items(): if has_arg(fn, name): res.update({name: value}) res.update(override) return res
def check_params(self, params): """Check for user typos in `params`. # Arguments params: dictionary; the parameters to be checked # Raises ValueError: if any member of `params` is not a valid argument. """ legal_params_fns = [ Sequential.fit, Sequential.fit_generator, Sequential.predict, Sequential.predict_classes, Sequential.evaluate, self.__model__, self.__callbacks__ ] for params_name in params: for fn in legal_params_fns: if has_arg(fn, params_name): break else: raise ValueError(f'{params_name} is not a legal parameter')
def _filter_n_set_params(self, fn, **params): """ Filter `params` and return fn(args). Arguments ---------- fn: function function to retrun. **params: dictonary. Dictionary of parameter names mapped to their values. Returns ---------- fn(expected args) """ args = {} for params_name in params: if has_arg(fn, params_name): args[params_name] = params[params_name] return fn(**args)
def _check_params(self, params, legal_params_fns): """ Checks for user typos in "params" for common functions. Arguments ---------- params: dictionary. the parameters to be checked. legal_params_fns: list of functions. the functions to be checked Raises ---------- ValueError: if any member of `params` is not a valid argument. """ for params_name in params: for fn in legal_params_fns: if has_arg(fn, params_name): break else: raise ValueError('{} is not a legal parameter'.format(params_name))
def call(self, inputs, training=None): kwargs = {} if has_arg(self.layer.call, 'training'): kwargs['training'] = training uses_learning_phase = False input_shape = K.int_shape(inputs) if False and input_shape[0]: # batch size matters, use rnn-based implementation def step(x, _): global uses_learning_phase output = self.layer.call(x, **kwargs) if hasattr(output, '_uses_learning_phase'): uses_learning_phase = (output._uses_learning_phase or uses_learning_phase) return output, [] _, outputs, _ = K.rnn(step, inputs, initial_states=[], input_length=input_shape[1], unroll=False) y = outputs else: ''' # No batch size specified, therefore the layer will be able # to process batches of any size. # We can go with reshape-based implementation for performance. input_length = input_shape[1] if not input_length: input_length = K.shape(inputs)[1] # Shape: (num_samples * timesteps, ...). And track the # transformation in self._input_map. input_uid = _object_list_uid(inputs) inputs = K.reshape(inputs, (-1,) + input_shape[2:]) self._input_map[input_uid] = inputs # (num_samples * timesteps, ...) y = self.layer.call(inputs, **kwargs) if hasattr(y, '_uses_learning_phase'): uses_learning_phase = y._uses_learning_phase # Shape: (num_samples, timesteps, ...) output_shape = self.compute_output_shape(input_shape) y = K.reshape(y, (-1, input_length) + output_shape[2:]) ''' input_length = input_shape[1] output_shape = self.compute_output_shape(input_shape) # self.gpuInputVar = tf.zeros( (input_shape[0],) + input_shape[2:]) self.gpuInputVar = None # with K.device('/cpu:0'): # self.cpuOutputVar = tf.zeros(output_shape) # tsteps = None # with K.device('/cpu:0'): # tsteps = K.unstack(inputs, axis=1) # outList = [] # ins = K.reshape(inputs, (input_length, input_shape[0]) + input_shape[2:]) numDims = len(input_shape) with K.device('/cpu:0'): ins = tf.transpose(inputs, (1, 0) + tuple([i for i in range(2, numDims)])) def foo(timestepPlusDim): self.gpuInputVar = timestepPlusDim # with tf.device('/gpu:0'): # self.gpuInputVar = tf.squeeze(timestepPlusDim) # with tf.device('/cpu:0'): # return self.layer.call(self.gpuInputVar) with tf.device('/gpu:0'): out = self.layer.call(self.gpuInputVar) with tf.device('/cpu:0'): return out with tf.device('/cpu:0'): y = tf.map_fn(foo, ins, back_prop=True, swap_memory=True) ''' # this should be on the device we are otherwise using... gpu... # for i, timestep in enumerate(tsteps): for timestep in tsteps: # out = None # with tf.device('/gpu:0'): # test = tsteps[i] # with tf.device('/cpu:0'): # gpuVar = tf.identity(tsteps[i]) gpuVar = timestep * 1 # with K.device('/cpu:0'): # out = self.layer.call(gpuVar, **kwargs) out = self.layer.call(gpuVar, **kwargs) with tf.device('/cpu:0'): cpuVar = out * 1 # outList.append(self.layer.call(tsteps[i] , **kwargs)) outList.append(cpuVar) # tsteps[i] = K.reshape(tsteps[i], (-1, 1) + output_shape[2:]) # ''' # ins = K.reshape(inputs, (-1,) + input_shape[2:]) # ins = tf.transpose(inputs, (1, 0,2,3)) # def test(timestep): # with tf.device('/cpu:0'): # return self.layer.call(timestep) # y = tf.map_fn(test, ins) # print(len(outList)) # with tf.device('/cpu:0'): # with K.device('/cpu:0'): # y = K.stack(outList, axis=1) # Apply activity regularizer if any: if (hasattr(self.layer, 'activity_regularizer') and self.layer.activity_regularizer is not None): regularization_loss = self.layer.activity_regularizer(y) self.add_loss(regularization_loss, inputs) if uses_learning_phase: y._uses_learning_phase = True return y # '''
def call(self, inputs, mask=None, training=None, initial_state=None, constants=None): # The input should be dense, padded with zeros. If a ragged input is fed # into the layer, it is padded and the row lengths are used for masking. inputs, row_lengths = backend.convert_inputs_if_ragged(inputs) is_ragged_input = (row_lengths is not None) self._validate_args_if_ragged(is_ragged_input, mask) inputs, initial_state, constants = self._process_inputs( inputs, initial_state, constants) self._maybe_reset_cell_dropout_mask(self.cell) if isinstance(self.cell, StackedRNNCells): for cell in self.cell.cells: self._maybe_reset_cell_dropout_mask(cell) if mask is not None: # Time step masks must be the same for each input. # TODO(scottzhu): Should we accept multiple different masks? mask = tf.nest.flatten(mask)[0] if tf.nest.is_nested(inputs): # In the case of nested input, use the first element for shape check. input_shape = backend.int_shape(tf.nest.flatten(inputs)[0]) else: input_shape = backend.int_shape(inputs) timesteps = input_shape[0] if self.time_major else input_shape[1] if self.unroll and timesteps is None: raise ValueError('Cannot unroll a RNN if the ' 'time dimension is undefined. \n' '- If using a Sequential model, ' 'specify the time dimension by passing ' 'an `input_shape` or `batch_input_shape` ' 'argument to your first layer. If your ' 'first layer is an Embedding, you can ' 'also use the `input_length` argument.\n' '- If using the functional API, specify ' 'the time dimension by passing a `shape` ' 'or `batch_shape` argument to your Input layer.') kwargs = {} if generic_utils.has_arg(self.cell.call, 'training'): kwargs['training'] = training # TF RNN cells expect single tensor as state instead of list wrapped tensor. is_tf_rnn_cell = getattr(self.cell, '_is_tf_rnn_cell', None) is not None # Use the __call__ function for callable objects, eg layers, so that it # will have the proper name scopes for the ops, etc. cell_call_fn = self.cell.__call__ if callable( self.cell) else self.cell.call if constants: if not generic_utils.has_arg(self.cell.call, 'constants'): raise ValueError( f'RNN cell {self.cell} does not support constants. ' f'Received: constants={constants}') def step(inputs, states): constants = states[-self._num_constants:] # pylint: disable=invalid-unary-operand-type states = states[:-self._num_constants] # pylint: disable=invalid-unary-operand-type states = states[0] if len( states) == 1 and is_tf_rnn_cell else states output, new_states = cell_call_fn(inputs, states, constants=constants, **kwargs) if not tf.nest.is_nested(new_states): new_states = [new_states] return output, new_states else: def step(inputs, states): states = states[0] if len( states) == 1 and is_tf_rnn_cell else states output, new_states = cell_call_fn(inputs, states, **kwargs) if not tf.nest.is_nested(new_states): new_states = [new_states] return output, new_states last_output, outputs, states = backend.rnn( step, inputs, initial_state, constants=constants, go_backwards=self.go_backwards, mask=mask, unroll=self.unroll, input_length=row_lengths if row_lengths is not None else timesteps, time_major=self.time_major, zero_output_for_mask=self.zero_output_for_mask) if self.stateful: updates = [ tf.compat.v1.assign(self_state, tf.cast(state, self_state.dtype)) for self_state, state in zip(tf.nest.flatten(self.states), tf.nest.flatten(states)) ] self.add_update(updates) if self.return_sequences: output = backend.maybe_convert_to_ragged( is_ragged_input, outputs, row_lengths, go_backwards=self.go_backwards) else: output = last_output if self.return_state: if not isinstance(states, (list, tuple)): states = [states] else: states = list(states) return generic_utils.to_list(output) + states else: return output
def call(self, inputs, mask=None, training=None, initial_state=None, constants=None): # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if isinstance(inputs, list): inputs = inputs[0] if initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if isinstance(mask, list): mask = mask[0] if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') timesteps = K.int_shape(inputs)[1] kwargs = {} if has_arg(self.cell.call, 'training'): kwargs['training'] = training if constants: if not has_arg(self.cell.call, 'constants'): raise ValueError('RNN cell does not support constants') def step(inputs, states): constants = states[-self._num_constants:] states = states[:-self._num_constants] return self.cell.call(inputs, states, constants=constants, **kwargs) else: def step(inputs, states): return self.cell.call(inputs, states, **kwargs) last_output, outputs, states = K.rnn(step, inputs, initial_state, constants=constants, go_backwards=self.go_backwards, mask=mask, input_length=timesteps) if self.stateful: updates = [] for i in range(len(states)): updates.append((self.states[i], states[i])) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = last_output # Properly set learning phase if getattr(last_output, '_uses_learning_phase', False): output._uses_learning_phase = True if self.return_state: states = to_list(states, allow_tuple=True) return [output] + states else: return output
def call(self, inputs, mask=None, training=None, initial_state=None): # input shape: `(samples, time (padded with zeros), input_dim)` # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if isinstance(inputs, list): initial_state = inputs[1:] inputs = inputs[0] elif initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if isinstance(mask, list): mask = mask[0] if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') input_shape = K.int_shape(inputs) print("input:{}".format(input_shape[1])) timesteps = input_shape[1] if self.unroll and timesteps in [None, 1]: raise ValueError('Cannot unroll a RNN if the ' 'time dimension is undefined or equal to 1. \n' '- If using a Sequential model, ' 'specify the time dimension by passing ' 'an `input_shape` or `batch_input_shape` ' 'argument to your first layer. If your ' 'first layer is an Embedding, you can ' 'also use the `input_length` argument.\n' '- If using the functional API, specify ' 'the time dimension by passing a `shape` ' 'or `batch_shape` argument to your Input layer.') if has_arg(self.cell.call, 'training'): step = functools.partial(self.cell.call, training=training) else: step = self.cell.call ''' last_output, outputs, states, sT = K_new.rnn(step, inputs, initial_state, go_backwards=self.go_backwards, mask=mask, unroll=self.unroll, input_length=timesteps) ''' last_output, outputs, states = K.rnn(step, inputs, initial_state, go_backwards=self.go_backwards, mask=mask, unroll=self.unroll, input_length=timesteps) print("OUTPUTS: {}, {}, {}".format(K.int_shape(last_output), K.int_shape(outputs), states)) if self.stateful: updates = [] for i in range(len(states)): updates.append((self.states[i], states[i])) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = last_output # Properly set learning phase if getattr(last_output, '_uses_learning_phase', False): output._uses_learning_phase = True if self.return_state: if not isinstance(states, (list, tuple)): states = [states] else: states = list(states) return [output] + list(states) else: return output
def call( self, inputs, training=None, mask=None, initial_state=None, constants=None, ): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, "training"): kwargs["training"] = training if generic_utils.has_arg(self.layer.call, "mask"): kwargs["mask"] = mask if generic_utils.has_arg(self.layer.call, "constants"): kwargs["constants"] = constants if generic_utils.has_arg(self.layer.call, "initial_state"): if isinstance(inputs, list) and len(inputs) > 1: # initial_states are keras tensors, which means they are passed # in together with inputs as list. The initial_states need to be # split into forward and backward section, and be feed to layers # accordingly. forward_inputs = [inputs[0]] backward_inputs = [inputs[0]] pivot = (len(inputs) - self._num_constants) // 2 + 1 # add forward initial state forward_inputs += inputs[1:pivot] if not self._num_constants: # add backward initial state backward_inputs += inputs[pivot:] else: # add backward initial state backward_inputs += inputs[pivot : -self._num_constants] # add constants for forward and backward layers forward_inputs += inputs[-self._num_constants :] backward_inputs += inputs[-self._num_constants :] forward_state, backward_state = None, None if "constants" in kwargs: kwargs["constants"] = None elif initial_state is not None: # initial_states are not keras tensors, eg eager tensor from np # array. They are only passed in from kwarg initial_state, and # should be passed to forward/backward layer via kwarg # initial_state as well. forward_inputs, backward_inputs = inputs, inputs half = len(initial_state) // 2 forward_state = initial_state[:half] backward_state = initial_state[half:] else: forward_inputs, backward_inputs = inputs, inputs forward_state, backward_state = None, None y = self.forward_layer( forward_inputs, initial_state=forward_state, **kwargs ) y_rev = self.backward_layer( backward_inputs, initial_state=backward_state, **kwargs ) else: y = self.forward_layer(inputs, **kwargs) y_rev = self.backward_layer(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: time_dim = ( 0 if getattr(self.forward_layer, "time_major", False) else 1 ) y_rev = backend.reverse(y_rev, time_dim) if self.merge_mode == "concat": output = backend.concatenate([y, y_rev]) elif self.merge_mode == "sum": output = y + y_rev elif self.merge_mode == "ave": output = (y + y_rev) / 2 elif self.merge_mode == "mul": output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] else: raise ValueError( "Unrecognized value for `merge_mode`. " f"Received: {self.merge_mode}" 'Expected values are ["concat", "sum", "ave", "mul"]' ) if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def test_has_arg_positional_only(): assert has_arg(pow, 'x') is False
def call(self, inputs, mask=None, training=None, initial_state=None, constants=None): # input shape: `(samples, time (padded with zeros), input_dim)` # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if isinstance(inputs, list): inputs = inputs[0] if initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if isinstance(mask, list): mask = mask[0] if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') input_shape = K.int_shape(inputs) timesteps = input_shape[1] if self.unroll and timesteps in [None, 1]: raise ValueError('Cannot unroll a RNN if the ' 'time dimension is undefined or equal to 1. \n' '- If using a Sequential model, ' 'specify the time dimension by passing ' 'an `input_shape` or `batch_input_shape` ' 'argument to your first layer. If your ' 'first layer is an Embedding, you can ' 'also use the `input_length` argument.\n' '- If using the functional API, specify ' 'the time dimension by passing a `shape` ' 'or `batch_shape` argument to your Input layer.') kwargs = {} if has_arg(self.cell.call, 'training'): kwargs['training'] = training #print(constants) #print(initial_state, 'in call function') #if constants: if constants is not None: if not has_arg(self.cell.call, 'constants'): raise ValueError('RNN cell does not support constants') if isinstance(constants, (list, tuple)): self._num_constants = len(constants) else: self._num_constants = 1 def step(inputs, states): #print(states,'states in step') #print(self._num_constants, '~~~') constants = states[-self._num_constants:] states = states[:-self._num_constants] return self.cell.call(inputs, states, constants=constants, **kwargs) else: def step(inputs, states): return self.cell.call(inputs, states, **kwargs) last_output, outputs, states = K.rnn(step, inputs, initial_state, constants=constants, go_backwards=self.go_backwards, mask=mask, unroll=self.unroll, input_length=timesteps) if self.stateful: updates = [] for i in range(len(states)): updates.append((self.states[i], states[i])) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = last_output # Properly set learning phase if getattr(last_output, '_uses_learning_phase', False): output._uses_learning_phase = True for state in states: state._uses_learning_phase = True if self.return_state: if not isinstance(states, (list, tuple)): states = [states] else: states = list(states) return [output] + states else: return output
def call(self, inputs, training=None, mask=None, initial_state=None, constants=None): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training if generic_utils.has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask if generic_utils.has_arg(self.layer.call, 'constants'): kwargs['constants'] = constants if generic_utils.has_arg(self.layer.call, 'initial_state'): if isinstance(inputs, list) and len(inputs) > 1: # initial_states are keras tensors, which means they are passed in # together with inputs as list. The initial_states need to be split into # forward and backward section, and be feed to layers accordingly. forward_inputs = [inputs[0]] backward_inputs = [inputs[0]] pivot = (len(inputs) - self._num_constants) // 2 + 1 # add forward initial state forward_inputs += inputs[1:pivot] if not self._num_constants: # add backward initial state backward_inputs += inputs[pivot:] else: # add backward initial state backward_inputs += inputs[pivot:-self._num_constants] # add constants for forward and backward layers forward_inputs += inputs[-self._num_constants:] backward_inputs += inputs[-self._num_constants:] forward_state, backward_state = None, None if 'constants' in kwargs: kwargs['constants'] = None elif initial_state is not None: # initial_states are not keras tensors, eg eager tensor from np array. # They are only passed in from kwarg initial_state, and should be passed # to forward/backward layer via kwarg initial_state as well. forward_inputs, backward_inputs = inputs, inputs half = len(initial_state) // 2 forward_state = initial_state[:half] backward_state = initial_state[half:] else: forward_inputs, backward_inputs = inputs, inputs forward_state, backward_state = None, None y = self.forward_layer(forward_inputs, initial_state=forward_state, **kwargs) y_rev = self.backward_layer(backward_inputs, initial_state=backward_state, **kwargs) else: y = self.forward_layer(inputs, **kwargs) y_rev = self.backward_layer(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: time_dim = 0 if getattr(self.forward_layer, 'time_major', False) else 1 y_rev = K.reverse(y_rev, time_dim) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] else: raise ValueError( 'Unrecognized value for `merge_mode`: %s' % (self.merge_mode)) if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def call(self, inputs, training=None, mask=None): kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training input_shape = tf.nest.map_structure( lambda x: tf.TensorShape(K.int_shape(x)), inputs) batch_size = tf_utils.convert_shapes(input_shape) batch_size = tf.nest.flatten(batch_size)[0] if batch_size and not self._always_use_reshape: inputs, row_lengths = K.convert_inputs_if_ragged(inputs) is_ragged_input = row_lengths is not None input_length = tf_utils.convert_shapes(input_shape) input_length = tf.nest.flatten(input_length)[1] # batch size matters, use rnn-based implementation def step(x, _): output = self.layer(x, **kwargs) return output, [] _, outputs, _ = K.rnn( step, inputs, initial_states=[], input_length=row_lengths[0] if is_ragged_input else input_length, mask=mask, unroll=False) # pylint: disable=g-long-lambda y = tf.nest.map_structure( lambda output: K.maybe_convert_to_ragged(is_ragged_input, output, row_lengths), outputs) else: # No batch size specified, therefore the layer will be able # to process batches of any size. # We can go with reshape-based implementation for performance. is_ragged_input = tf.nest.map_structure( lambda x: isinstance(x, tf.RaggedTensor), inputs) is_ragged_input = tf.nest.flatten(is_ragged_input) if all(is_ragged_input): input_values = tf.nest.map_structure(lambda x: x.values, inputs) input_row_lenghts = tf.nest.map_structure( lambda x: x.nested_row_lengths()[0], inputs) y = self.layer(input_values, **kwargs) y = tf.nest.map_structure(tf.RaggedTensor.from_row_lengths, y, input_row_lenghts) elif any(is_ragged_input): raise ValueError('All inputs has to be either ragged or not, ' 'but not mixed. You passed: {}'.format(inputs)) else: input_length = tf_utils.convert_shapes(input_shape) input_length = tf.nest.flatten(input_length)[1] if not input_length: input_length = tf.nest.map_structure(lambda x: tf.compat.v1.shape(x)[1], inputs) input_length = generic_utils.to_list(tf.nest.flatten(input_length))[0] inner_input_shape = tf.nest.map_structure( lambda x: self._get_shape_tuple((-1,), x, 2), inputs) # Shape: (num_samples * timesteps, ...). And track the # transformation in self._input_map. inputs = tf.__internal__.nest.map_structure_up_to(inputs, tf.reshape, inputs, inner_input_shape) # (num_samples * timesteps, ...) if generic_utils.has_arg(self.layer.call, 'mask') and mask is not None: inner_mask_shape = self._get_shape_tuple((-1,), mask, 2) kwargs['mask'] = K.reshape(mask, inner_mask_shape) y = self.layer(inputs, **kwargs) # Shape: (num_samples, timesteps, ...) output_shape = self.compute_output_shape(input_shape) # pylint: disable=g-long-lambda output_shape = tf.nest.map_structure( lambda tensor, int_shape: self._get_shape_tuple( (-1, input_length), tensor, 1, int_shape[2:]), y, output_shape) y = tf.__internal__.nest.map_structure_up_to(y, tf.reshape, y, output_shape) if not tf.executing_eagerly(): # Set the static shape for the result since it might be lost during # array_ops reshape, eg, some `None` dim in the result could be # inferred. tf.__internal__.nest.map_structure_up_to( y, lambda tensor, shape: tensor.set_shape(shape), y, self.compute_output_shape(input_shape)) return y
def call(self, inputs, training=None, mask=None): if not isinstance(inputs, list): return super(TimeDistributedMultiInput, self).call(inputs, training=training, mask=mask) kwargs = {} if has_arg(self.layer.call, 'training'): kwargs['training'] = training uses_learning_phase = False input_shapes = [K.int_shape(inp) for inp in inputs] batch_sizes = [shape[0] for shape in input_shapes if shape is not None] fixed_batch_size = any([bs is not None for bs in batch_sizes]) if fixed_batch_size: # batch size matters, use rnn-based implementation def step(x, _): global uses_learning_phase output = self.layer.call(x, **kwargs) if hasattr(output, '_uses_learning_phase'): uses_learning_phase = (output._uses_learning_phase or uses_learning_phase) return output, [] # Note: will likely fail here if K.rnn doesn't like multiple inputs _, outputs, _ = K.rnn(step, inputs, initial_states=[], input_length=input_shapes[1], unroll=False) y = outputs else: # No batch size specified, therefore the layer will be able # to process batches of any size. # We can go with reshape-based implementation for performance. input_length = self.timesteps if self.timesteps else K.shape(inputs[0])[1] # ^^ assumes input 0 has the correct number of timesteps def prep_input(inp): inner_input_shape = self._get_shape_tuple((-1,), inp, 2) # Shape: (num_samples * timesteps, ...). And track the # transformation in self._input_map. input_uid = object_list_uid(inp) reshaped = K.reshape(inp, inner_input_shape) self._input_map[input_uid] = reshaped return reshaped inputs = [prep_input(inp) for inp in inputs] # (num_samples * timesteps, ...) if has_arg(self.layer.call, 'mask') and mask is not None: inner_mask_shape = self._get_shape_tuple((-1,), mask, 2) kwargs['mask'] = K.reshape(mask, inner_mask_shape) y = self.layer.call(inputs, **kwargs) if isinstance(y, list): raise NotImplementedError( 'TimeDistributedMultiInput not implemented for multiple ' 'output tensors yet.') if hasattr(y, '_uses_learning_phase'): uses_learning_phase = y._uses_learning_phase # Shape: (num_samples, timesteps, ...) output_shape = self.compute_output_shape(input_shapes) output_shape = self._get_shape_tuple( (-1, input_length), y, 1, output_shape[2:]) y = K.reshape(y, output_shape) # Apply activity regularizer if any: if (hasattr(self.layer, 'activity_regularizer') and self.layer.activity_regularizer is not None): regularization_loss = self.layer.activity_regularizer(y) self.add_loss(regularization_loss, inputs) if uses_learning_phase: y._uses_learning_phase = True return y