import tensorflow.python as tf
from tensorflow.python import keras
import numpy as np
# keras.backend.clear_session()
tf.enable_eager_execution()
x = keras.Input(shape=(10))
label_1 = keras.Input(shape=(10))
label_2 = keras.Input(shape=(10))
pred_1 = keras.layers.Dense(10)(x)
pred_2 = keras.layers.Dense(10)(x)
model = keras.Model(inputs=[x, label_1, label_2], outputs=[pred_1, pred_2])
model.summary()
train_set = tf.data.Dataset.from_tensor_slices({
'input_1':
np.random.randint(1, 4, 1000).reshape(100, 10),
'input_2':
np.random.randint(5, 8, 1000).reshape(100, 10),
'input_3':
np.random.randint(9, 12, 1000).reshape(100, 10)
}).repeat().batch(32)
train_set = train_set.map(
lambda x: {
'input_1': float(x['input_1']) + .1,
'input_2': float(x['input_2']) - .2,
'input_3': float(x['input_3']) - .5
})
from custom.layers import *
from custom.callback import *
import numpy as np
from tensorflow.python import keras, enable_eager_execution
tf.executing_eagerly()
enable_eager_execution()
class MusicTransformerV2:
def __init__(self, embedding_dim = 256, vocab_size =240, num_layer =6,
max_seq = 2048,l_r = 0.001, debug = False, dropout = 0.1):
self._debug = debug
self.num_layer = num_layer
self.embedding_dim = embedding_dim
self.vocab_size = vocab_size
self.max_seq = max_seq
self.dropout = dropout
self.model = self._build_model()
optim = keras.optimizers.Adam(lr=l_r, beta_1=0.9, beta_2=0.98, epsilon=1e-9)
# loss_func = TransformerLoss()
self.model.compile(optim, loss='categorical_crossentropy', metrics = ['accuracy'])
pass
def _decoder(self, input_tensor, layer=0):
if self._debug:
print('[DEBUG]:{}'.format('decoder called'))
decoder1 = RelativeGlobalAttention(64)([input_tensor, input_tensor, input_tensor])# Assuming Dh = 64
decoder1 = keras.layers.Dropout(rate=self.dropout)(decoder1)
add_and_norm = keras.layers.Add()([decoder1, input_tensor])
add_and_norm = keras.layers.LayerNormalization()(add_and_norm)