def create_state_var(self, name, initial_value=None, data_shape=None, choice_dependent=None): """ A state var is a variable where the initial value is given by the encoder, or a constant, and the final value is determined by one step of this rec layer (usually called the decoder). :param str name: :param tf.Tensor|None initial_value: assumes batch-major, if data_shape is not given :param Data|None data_shape: :param bool|None choice_dependent: choice dependent or not relevant :rtype: tf.Tensor """ assert name not in self.state_vars assert data_shape or initial_value is not None if data_shape: assert isinstance(data_shape, Data) elif initial_value.shape.ndims == 0: data_shape = Data(name=name, batch_dim_axis=None, shape=(), dtype=initial_value.dtype.name) else: assert initial_value.shape.dims[0].value is None # first is batch dim data_shape = Data( name=name, batch_dim_axis=0, shape=initial_value.shape.as_list()[1:], dtype=initial_value.dtype.name) if initial_value is not None: # initial_value might have dim 1 in variable dimensions (which are not the batch-dim-axis), # see get_rec_initial_output, which should be fine for broadcasting. initial_value.set_shape(data_shape.batch_shape) var = self.StateVar(parent=self, name=name, initial_value=initial_value, data_shape=data_shape, choice_dependent=choice_dependent) self.state_vars[name] = var return var.read()
def __init__(self, name, network, n_out=None, out_type=None, sources=(), target=None, loss=None, loss_opts=None, L2=None, is_output_layer=None, trainable=True): """ :param str name: :param TFNetwork.TFNetwork network: :param None|int n_out: output dim :param dict[str] out_type: kwargs for Data class. more explicit than n_out. :param list[LayerBase] sources: :param str|None target: if some loss is set, this is the target data-key, i.e. network.extern_data.get_data(target) alternatively, this also can be a layer name. :param str|None loss: if set, via get_loss :param dict[str]|None loss_opts: kwargs for Loss class, if loss is set :param float|None L2: for constraints :param bool|None is_output_layer: :param bool trainable: mostly ignored for now... """ self.name = name self.network = network if loss and not target: target = self.network.extern_data.default_target self.target = target self.loss = None # type: Loss if loss: loss_class = get_loss_class(loss) self.loss = loss_class(**(loss_opts or {})) if self.loss.recurrent: self.recurrent = True if out_type is None and n_out is None and target: n_out = self._get_target_value(mark_data_key_as_used=False).dim if self.loss: n_out = self.loss.get_auto_output_layer_dim(n_out) if out_type is None: assert n_out out_type = {"dim": n_out} out_type = out_type.copy() out_type.setdefault("name", "%s_output" % self.name) if n_out is not None: out_type.setdefault("dim", n_out) assert out_type["dim"] == n_out # You are supposed to set self.output.{batch_dim_axis,time_dim_axis} explicitly, # as well as check the inputs if they are as you would suggest. # However, a good default is often to use the same as the input. if sources and "batch_dim_axis" not in out_type: out_type.setdefault("batch_dim_axis", sources[0].output.batch_dim_axis) out_type.setdefault("time_dim_axis", sources[0].output.time_dim_axis) self.output = Data(**out_type) # You are supposed to set self.output.placeholder to the value which you want to return by the layer. # Normally you are also supposed to set self.output.size_placeholder explicitly, just like self.output.placeholder. # However, in many cases, this will just be {0: time-lengths} and the same as from the input. # We check for this case and preset it by that if possible. # If you want to have it different in your layer, just overwrite it. if sources and sources[0].output.matches_dim_pattern(self.output): self.output.size_placeholder = sources[0].output.size_placeholder.copy() self.output_before_activation = None # type: None|OutputWithActivation self.sources = sources self.params = {} # type: dict[str,tf.Variable] self.L2 = L2 self._is_output_layer = is_output_layer self.trainable = trainable
def get_out_data_from_opts(cls, name, sources, pool_size, n_out=None, **kwargs): input_data = get_concat_sources_data_template(sources) assert not input_data.sparse return Data( name="%s_output" % name, shape=[ input_data.get_placeholder_as_batch_major().shape[1].value, input_data.get_placeholder_as_batch_major().shape[2].value ], dtype=input_data.dtype, size_placeholder={ 0: tf.strided_slice( input_data.size_placeholder[ input_data.time_dim_axis_excluding_batch], [0], tf.shape(input_data.size_placeholder[ input_data.time_dim_axis_excluding_batch]), [pool_size]) }, sparse=False, batch_dim_axis=0, time_dim_axis=1)
def get_out_data_from_opts(cls, name, sources, nr_of_channels, n_out=None, **kwargs): input_data = get_concat_sources_data_template(sources) assert not input_data.sparse return Data( name="%s_output" % name, shape=[ input_data.get_placeholder_as_batch_major().shape[1].value, input_data.get_placeholder_as_batch_major().shape[2].value // nr_of_channels ], dtype=input_data.dtype, size_placeholder={ 0: tf.reshape( tf.tile( tf.reshape( input_data.size_placeholder[ input_data.time_dim_axis_excluding_batch], [-1, 1]), [1, nr_of_channels]), [-1]) }, sparse=False, batch_dim_axis=0, time_dim_axis=1)
def __init__(self, axis=None, axis_kind=None, slice_start=None, slice_end=None, slice_step=None, **kwargs): """ :param int|None axis: :param str|None axis_kind: "T" for time, "B" for batch, "F" for feature :param int|None slice_start: :param int|None slice_end: :param int|None slice_step: :param int|None n_out: """ # Dummy out_type for now, will reset layer. super(SliceLayer, self).__init__(out_type={"shape": ()}, **kwargs) if axis is not None: assert not axis_kind assert 0 <= axis < len(self.input_data.batch_shape) else: assert axis_kind axis_kind = axis_kind.upper() if axis_kind == "T": assert self.input_data.time_dim_axis is not None axis = self.input_data.time_dim_axis elif axis_kind == "B": assert self.input_data.batch_dim_axis is not None axis = self.input_data.batch_dim_axis elif axis_kind == "F": axes = self.input_data.get_axes(exclude_time=True, exclude_batch=True) assert len(axes) == 1 axis = axes[0] dim_slice = slice(slice_start, slice_end, slice_step) slices = [slice(None, None)] * axis + [dim_slice] out_type = self.input_data.get_kwargs() axis_wo_batch = self.input_data.get_batch_axis_excluding_batch(axis) if axis_wo_batch is not None: out_type["shape"] = list(out_type["shape"]) if out_type["shape"][axis_wo_batch] is not None: out_type["shape"][axis_wo_batch] = len(range(out_type["shape"][axis_wo_batch])[dim_slice]) if axis_wo_batch == len(out_type["shape"]) - 1 and not out_type["sparse"]: out_type["dim"] = out_type["shape"][axis_wo_batch] self.output = Data(**out_type) self.output.size_placeholder = self.input_data.size_placeholder if axis == self.input_data.time_dim_axis: if slice_start: assert slice_start > 0 self.output.size_placeholder[self.input_data.time_dim_axis_excluding_batch] = \ tf.maximum(0, self.output.size_placeholder[self.input_data.time_dim_axis_excluding_batch] - slice_start) if slice_end: assert slice_end > 0 self.output.size_placeholder[self.input_data.time_dim_axis_excluding_batch] = \ tf.minimum( tf.shape(self.input_data.placeholder)[self.input_data.time_dim_axis] - slice_end, self.output.size_placeholder[self.input_data.time_dim_axis_excluding_batch]) if slice_step: self.output.size_placeholder[self.input_data.time_dim_axis_excluding_batch] //= slice_step elif axis_wo_batch is not None: assert axis_wo_batch not in self.output.size_placeholder self.output.placeholder = self.input_data.placeholder[slices]
def targetb_linear_out(sources, **kwargs): from TFUtil import Data enc = sources[1].output dec = sources[0].output size = enc.get_sequence_lengths() # + dec.get_sequence_lengths() # output_len_tag.set_tag_on_size_tensor(size) return Data(name="targetb_linear", sparse=True, dim=eval("targetb_num_labels"), size_placeholder={0: size})
def concat_sources(src_layers): """ :param list[LayerBase] src_layers: :return: data with placeholders set :rtype: Data """ assert src_layers, "need source layers" if len(src_layers) == 1: return src_layers[0].output assert not src_layers[0].output.sparse, "sparse concat not supported" shape = src_layers[0].output.shape # without batch-dim assert shape, "source must not be a scalar of layer %r" % src_layers[0] prefix_shape = shape[:-1] dim = 0 dtype = src_layers[0].output.dtype batch_dim_axis = src_layers[0].output.batch_dim_axis time_dim_axis = src_layers[0].output.time_dim_axis for layer in src_layers: assert layer.output.dtype == dtype, "incompatible dtype with layer %r" % layer assert layer.output.batch_dim_axis == batch_dim_axis assert layer.output.time_dim_axis == time_dim_axis shape = layer.output.shape assert layer.output.placeholder.get_shape().ndims == len(shape) + 1 # with batch-dim assert shape, "source must not be a scalar of layer %r" % layer assert shape[:-1] == prefix_shape, "incompatible concat with layer %r" % layer assert shape[-1], "source last-dim must be specified of layer %r" % layer dim += shape[-1] data = Data( name="concat_sources", shape=prefix_shape + (dim,), dim=dim, sparse=False, batch_dim_axis=batch_dim_axis, time_dim_axis=time_dim_axis, dtype=dtype) data.placeholder = tf.concat( concat_dim=len(prefix_shape) + 1, # one more because this is with batch-dim values=[layer.output.placeholder for layer in src_layers]) data.size_placeholder = src_layers[0].output.size_placeholder.copy() return data
def get_out_data_from_opts(cls, name, sources, nr_of_channels, n_out=None, **kwargs): input_data = get_concat_sources_data_template( sources).copy_as_batch_major() assert not input_data.sparse return Data(name="%s_output" % name, shape=[ input_data.batch_shape[1], input_data.batch_shape[2] // nr_of_channels ], dtype=input_data.dtype, sparse=False, batch_dim_axis=0, time_dim_axis=1)
def get_out_data_from_opts(cls, n_out=NotSpecified, out_type=None, sources=(), **kwargs): """ :param int|None|NotSpecified n_out: :param dict[str]|None out_type: :param list[LayerBase] sources: :rtype: Data """ out_type_ = {} if sources and any(sources): out_type_.update( Data.get_common_data([s.output for s in sources if s]).get_kwargs()) if n_out is not NotSpecified: out_type_["dim"] = n_out out_type_["name"] = "%s_output" % kwargs["name"] if out_type: if isinstance(out_type, dict): out_type_.update(out_type) elif callable(out_type): out_type_ = out_type # just overwrite else: raise TypeError("unexpected type of out_type %r" % (out_type, )) ######## ADDED ############ #out_type_["batch_dim_axis"] = 0 #out_type_["feature_dim_axis"] = 1 #out_type_["time_dim_axis"] = 2 ########################### return super(GatedRecurrentContextLayer, cls).get_out_data_from_opts(n_out=n_out, out_type=out_type_, sources=sources, **kwargs)
def get_out_data_from_opts(cls, name, sources, repetitions, n_out=None, **kwargs): input_data = get_concat_sources_data_template(sources) assert not input_data.sparse return Data( name="%s_output" % name, shape=[ input_data.get_placeholder_as_batch_major().shape[1].value, input_data.get_placeholder_as_batch_major().shape[2].value * repetitions ], dtype=input_data.dtype, sparse=False, size_placeholder={ 0: input_data.size_placeholder[ input_data.time_dim_axis_excluding_batch] }, batch_dim_axis=0, time_dim_axis=1)
def get_out_data_from_opts(cls, name, **kwargs): from TFUtil import Data return Data(name="%s_output" % name, batch_dim_axis=None, shape=(), dtype="float32") # scalar
class LayerBase(object): layer_class = None recurrent = False def __init__(self, name, network, n_out=None, out_type=None, sources=(), target=None, loss=None, loss_opts=None, L2=None, is_output_layer=None, trainable=True): """ :param str name: :param TFNetwork.TFNetwork network: :param None|int n_out: output dim :param dict[str] out_type: kwargs for Data class. more explicit than n_out. :param list[LayerBase] sources: :param str|None target: if some loss is set, this is the target data-key, i.e. network.extern_data.get_data(target) alternatively, this also can be a layer name. :param str|None loss: if set, via get_loss :param dict[str]|None loss_opts: kwargs for Loss class, if loss is set :param float|None L2: for constraints :param bool|None is_output_layer: :param bool trainable: mostly ignored for now... """ self.name = name self.network = network if loss and not target: target = self.network.extern_data.default_target self.target = target self.loss = None # type: Loss if loss: loss_class = get_loss_class(loss) self.loss = loss_class(**(loss_opts or {})) if self.loss.recurrent: self.recurrent = True if out_type is None and n_out is None and target: n_out = self._get_target_value(mark_data_key_as_used=False).dim if self.loss: n_out = self.loss.get_auto_output_layer_dim(n_out) if out_type is None: assert n_out out_type = {"dim": n_out} out_type = out_type.copy() out_type.setdefault("name", "%s_output" % self.name) if n_out is not None: out_type.setdefault("dim", n_out) assert out_type["dim"] == n_out # You are supposed to set self.output.{batch_dim_axis,time_dim_axis} explicitly, # as well as check the inputs if they are as you would suggest. # However, a good default is often to use the same as the input. if sources and "batch_dim_axis" not in out_type: out_type.setdefault("batch_dim_axis", sources[0].output.batch_dim_axis) out_type.setdefault("time_dim_axis", sources[0].output.time_dim_axis) self.output = Data(**out_type) # You are supposed to set self.output.placeholder to the value which you want to return by the layer. # Normally you are also supposed to set self.output.size_placeholder explicitly, just like self.output.placeholder. # However, in many cases, this will just be {0: time-lengths} and the same as from the input. # We check for this case and preset it by that if possible. # If you want to have it different in your layer, just overwrite it. if sources and sources[0].output.matches_dim_pattern(self.output): self.output.size_placeholder = sources[0].output.size_placeholder.copy() self.output_before_activation = None # type: None|OutputWithActivation self.sources = sources self.params = {} # type: dict[str,tf.Variable] self.L2 = L2 self._is_output_layer = is_output_layer self.trainable = trainable def __repr__(self): return "%s{class=%s, out_type=%s}" % ( self.name, self.layer_class, self.output.get_description(with_name=False)) @classmethod def cls_get_tf_scope_name(cls, name): """ :param str name: layer name :return: scope name, might be just name """ return name.replace(":", "__") @property def tf_scope_name(self): return self.cls_get_tf_scope_name(name=self.name) def is_output_layer(self): """ Some code differs between an output layer and other layers. It is a bit arbitrary what we define as output layer. :rtype: bool """ if self._is_output_layer is not None: return self._is_output_layer if self.target: return True if self.name == "output": return True return False def add_param(self, param): """ :param tf.Variable param: :return: param :rtype tf.Variable """ assert param.name self.params[param.name] = param return param def set_param_values_by_dict(self, values_dict, session): """ :param dict[str,numpy.ndarray] values_dict: :param tf.Session session: """ for param_name, values in values_dict.items(): param = self.params[param_name] assert isinstance(param, tf.Variable) shape = param.get_shape() assert isinstance(shape, tf.TensorShape) assert shape.is_fully_defined() assert tuple(shape.as_list()) == values.shape self.network.get_var_assigner(param).assign(values, session=session) def get_param_values_dict(self, session): """ :param tf.Session session: :return: dict name -> values :rtype: dict[str,numpy.ndarray] """ d = {} for param_name, param in self.params.items(): d[param_name] = param.eval(session) return d def _get_target_value(self, mark_data_key_as_used=True): """ :param bool mark_data_key_as_used: forwarded self.network.get_extern_data() :rtype: Data | None """ if not self.target or self.target == "none": return None if self.network.extern_data.has_data(self.target): return self.network.get_extern_data(self.target, mark_data_key_as_used=mark_data_key_as_used) if self.target in self.network.layers: return self.network.layers[self.target].output raise Exception("target %r unknown" % self.target) def _init_loss(self): if self.loss.output is self.output: return self.loss.init( output=self.output, output_with_activation=self.output_before_activation, target=self._get_target_value()) def get_loss_value(self): """ :return: the loss, a scalar value, or None if not set :rtype: tf.Tensor | None """ if not self.loss: return None self._init_loss() with tf.name_scope("loss"): return self.loss.get_value() def get_error_value(self): """ :return: usually the frame error rate, or None if not defined :rtype: tf.Tensor | None """ if not self.loss: return None self._init_loss() with tf.name_scope("error"): return self.loss.get_error() def get_params_l2_norm(self): return 2 * sum([tf.nn.l2_loss(param) for (name, param) in sorted(self.params.items())]) def get_constraints_value(self): c = 0 if self.L2: c += self.L2 * self.get_params_l2_norm() if c is 0: return None return c