def build(self, input_shape): assert isinstance(input_shape, list) and 2 == len(input_shape) out_shape, residual_shape = input_shape self.input_spec = [ keras.layers.InputSpec(shape=out_shape), keras.layers.InputSpec(shape=residual_shape) ] self.dense = keras.layers.Dense( units=self.params.hidden_size, kernel_initializer=self.create_initializer(), name="dense") self.dropout = keras.layers.Dropout(rate=self.params.hidden_dropout) self.layer_norm = LayerNormalization(name="LayerNorm") if self.params.adapter_size is not None: self.adapter_down = keras.layers.Dense( units=self.params.adapter_size, kernel_initializer=tf.keras.initializers.TruncatedNormal( stddev=self.params.adapter_init_scale), activation=self.get_activation(self.params.adapter_activation), name="adapter-down") self.adapter_up = keras.layers.Dense( units=self.params.hidden_size, kernel_initializer=tf.keras.initializers.TruncatedNormal( stddev=self.params.adapter_init_scale), name="adapter-up") super(ProjectionLayer, self).build(input_shape)
def test_seq_model(self): model = keras.Sequential([CustomLayer(num_units=17), LayerNormalization()]) model.compute_output_shape(input_shape=(16, 3, 4)) # model.build(input_shape=(16, 3, 4)) model.compile(optimizer='adam', loss='mse') model.fit(tf.ones((16, 3, 4), dtype=tf.float32), tf.ones((16, 3, 17), dtype=tf.float32), steps_per_epoch=2, epochs=10, callbacks=[pf.utils.create_one_cycle_lr_scheduler( max_learn_rate=5e-2, end_learn_rate=1e-7, warmup_epoch_count=5, total_epoch_count=10) ]) model.summary()
def test_equal(self): norm_layer = LayerNormalization() model = keras.Sequential( [keras.layers.InputLayer(input_shape=(16, 256)), norm_layer]) # model.build(input_shape=(3, 16, 256)) model.compile(optimizer=keras.optimizers.Adam(), loss='mse') # model.summary() model.fit(np.zeros((3, 16, 256)), np.ones((3, 16, 256))) model.summary() inputs = np.zeros((3, 16, 256)) predicted = model.predict(inputs) expected = np.ones_like(inputs) np.allclose(expected, predicted)
def build(self, input_shape): assert isinstance(input_shape, list) and 2 == len(input_shape) out_shape, residual_shape = input_shape self.input_spec = [ keras.layers.InputSpec(shape=out_shape), keras.layers.InputSpec(shape=residual_shape) ] self.dense = keras.layers.Dense( units=self.params.hidden_size, kernel_initializer=self.create_initializer(), name="dense") self.dropout = keras.layers.Dropout(rate=self.params.hidden_dropout) self.layer_norm = LayerNormalization(name="LayerNorm") super(ProjectionLayer, self).build(input_shape)
def test_serialization(self): model = keras.Sequential([LayerNormalization(input_shape=(2, 3))]) model.compile(optimizer='adam', loss='mse') model.summary() with tempfile.TemporaryDirectory() as temp_dir: temp_file = os.path.join(temp_dir, "model") model.save(temp_file) model = keras.models.load_model( temp_file, custom_objects={"LayerNormalization": LayerNormalization}) model.summary() encoded = model.to_json() model = keras.models.model_from_json( encoded, custom_objects={"LayerNormalization": LayerNormalization}) model.summary()
def test_serialization(self): model = keras.Sequential([LayerNormalization(input_shape=(2, 3))]) model.compile(optimizer='adam', loss='mse') model.summary() with tempfile.NamedTemporaryFile() as temp_file: temp_file.file.close() model.save(temp_file.name) model = keras.models.load_model( temp_file.name, custom_objects={"LayerNormalization": LayerNormalization}) model.summary() encoded = model.to_json() model = keras.models.model_from_json( encoded, custom_objects={"LayerNormalization": LayerNormalization}) model.summary()
def test_simple(self): norm_layer = LayerNormalization() model = keras.Sequential( [keras.layers.InputLayer(input_shape=(2, 3)), norm_layer]) model.build(input_shape=(None, 2, 3)) model.compile(optimizer=keras.optimizers.Adam(), loss='mse') model.summary() inputs = np.array([[[.2, .1, .3], [.5, .1, .1]]]) predict = model.predict(inputs) expected = np.asarray([[[0, -1.2247, 1.2247], [1.4142, -0.7071, -0.7071]]]) self.assertTrue(np.allclose(predict, expected, atol=1e-4))
def __init__(self, config, num_heads, x_dim, trainable=True, name=None, dtype=None, dynamic=False, **kwargs): super().__init__(trainable, name, dtype, dynamic, **kwargs) sa_dim = config.get('sa_dim', None) if sa_dim: self.shrink = tf.keras.layers.Dense(sa_dim, name='shrink') x_dim = sa_dim self.mlp_arc_h = MLP(n_hidden=config.n_mlp_arc, dropout=config.mlp_dropout, name='mlp_arc_h') self.mlp_arc_d = MLP(n_hidden=config.n_mlp_arc, dropout=config.mlp_dropout, name='mlp_arc_d') self.mlp_rel_h = MLP(n_hidden=config.n_mlp_rel, dropout=config.mlp_dropout, name='mlp_rel_h') self.mlp_rel_d = MLP(n_hidden=config.n_mlp_rel, dropout=config.mlp_dropout, name='mlp_rel_d') # the Biaffine layers self.arc_attn = Biaffine(n_in=config.n_mlp_arc, bias_x=True, bias_y=False, name='arc_attn') self.rel_attn = Biaffine(n_in=config.n_mlp_rel, n_out=config.n_rels, bias_x=True, bias_y=True, name='rel_attn') self.heads_WV = self.add_weight(shape=[num_heads, x_dim, x_dim]) self.dense = tf.keras.layers.Dense(x_dim) self.layer_norm = LayerNormalization(name="LayerNorm") self.graph = config.get('graph', False)
def layer_norm(name=None): """Return layer normalization function.""" return LayerNormalization(name=name)
def _construct(self, params): self.layer = CustomLayer.from_params(params) self.norm = LayerNormalization() self.supports_masking = True