def train_on_batch(self,
x,
y=None,
sample_weight=None,
class_weight=None,
reset_metrics=True):
if self._debug:
tf.print('sanity:\n',
self.sanity_check(x, y, mode='d'),
output_stream=sys.stdout)
x, y = self.__prepare_train_data(x, y)
_, _, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, x)
if self.dist:
predictions = self.__dist_train_step(x, y, look_ahead_mask, True)
else:
predictions = self.__train_step(x, y, look_ahead_mask, True)
if self._debug:
print('train step finished')
result_metric = []
if self.dist:
loss = self._distribution_strategy.reduce(
tf.distribute.ReduceOp.MEAN, self.loss_value, None)
else:
loss = tf.reduce_mean(self.loss_value)
loss = tf.reduce_mean(loss)
for metric in self.custom_metrics:
result_metric.append(metric(y, predictions).numpy())
return [loss.numpy()] + result_metric
def sanity_check(self, x, y, mode='v'):
# mode: v -> vector, d -> dict
x, inp_tar, out_tar = MusicTransformer.__prepare_train_data(x, y)
enc_mask, tar_mask, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, inp_tar)
predictions = self.call(x,
targets=inp_tar,
src_mask=enc_mask,
trg_mask=tar_mask,
lookup_mask=look_ahead_mask,
training=False)
if mode == 'v':
return predictions
elif mode == 'd':
dic = {}
for row in tf.argmax(predictions, -1).numpy():
for col in row:
try:
dic[str(col)] += 1
except KeyError:
dic[str(col)] = 1
return dic
else:
return tf.argmax(predictions, -1)
def generate(self, prior: list, beam=None, length=2048):
prior = tf.constant([prior])
decode_array = [par.token_sos]
decode_array = tf.constant([decode_array])
print(decode_array)
# TODO: add beam search
if beam is not None:
pass
else:
for i in range(min(self.max_seq, length)):
if i % 100 == 0:
print('generating... {}% completed'.format(
(i / min(self.max_seq, length)) * 100))
enc_mask, tar_mask, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], prior, decode_array)
result = self.call(prior,
targets=decode_array,
src_mask=enc_mask,
trg_mask=tar_mask,
lookup_mask=look_ahead_mask,
training=False)
result = tf.argmax(result, -1)
result = tf.cast(result, tf.int32)
decode_array = tf.concat(
[decode_array,
tf.expand_dims(result[:, -1], 0)], -1)
del enc_mask
del tar_mask
del look_ahead_mask
return decode_array.numpy()
def evaluate(self,
x=None,
y=None,
batch_size=None,
verbose=1,
sample_weight=None,
steps=None,
callbacks=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False):
x, inp_tar, out_tar = MusicTransformer.__prepare_train_data(x, y)
enc_mask, tar_mask, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, inp_tar)
predictions, weights = self.call(x,
targets=inp_tar,
src_mask=enc_mask,
trg_mask=tar_mask,
lookup_mask=look_ahead_mask,
training=False,
eval=True)
loss = tf.reduce_mean(self.loss(out_tar, predictions))
result_metric = []
for metric in self.custom_metrics:
result_metric.append(
metric(out_tar, tf.nn.softmax(predictions)).numpy())
return [loss.numpy()] + result_metric, weights
def generate(self, prior: list, length=2048, temperature=0.0):
decode_array = prior
decode_array = tf.constant([decode_array])
for i in range(min(self.max_seq, length)):
if decode_array.shape[1] >= self.max_seq:
break
print('generating... {:.1f}% completed'.format(
(i / min(self.max_seq, length)) * 100),
end="\r")
_, _, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], decode_array, decode_array, self.pad_token)
result, _ = self.call(decode_array,
lookup_mask=look_ahead_mask,
training=False,
eval=True)
result = result[:, -1] / (temperature + 1e-6)
result = tf.nn.softmax(result)
pdf = tfp.distributions.Categorical(probs=result)
result = pdf.sample(1)
result = tf.transpose(result, (1, 0))
result = tf.cast(result, tf.int32)
decode_array = tf.concat([decode_array, result], -1)
del look_ahead_mask
decode_array = decode_array[0]
return decode_array.numpy()
def train_on_batch(self,
x,
y=None,
sample_weight=None,
class_weight=None,
reset_metrics=True):
if self._debug:
tf.print('sanity:\n',
self.sanity_check(x, y, mode='d'),
output_stream=sys.stdout)
x, dec_input, target = MusicTransformer.__prepare_data(x, y)
enc_mask, tar_mask, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, dec_input)
# predictions = self.__train_step(x, dec_input, target, enc_mask, tar_mask, look_ahead_mask, True)
predictions = self.__dist_train_step(x, dec_input, target, enc_mask,
tar_mask, look_ahead_mask, True)
if self._debug:
print('train step finished')
result_metric = []
# loss = tf.reduce_mean(self.loss(target, predictions))
# loss = self.loss(target, predictions)
loss = self._distribution_strategy.reduce(tf.distribute.ReduceOp.MEAN,
self.loss_value, None)
loss = tf.reduce_mean(loss)
# loss = self._distribution_strategy.reduce(tf.distribute.ReduceOp.MEAN, loss, axis=-1)
for metric in self.custom_metrics:
result_metric.append(metric(target, predictions).numpy())
return [loss.numpy()] + result_metric
def generate(self,
decode_fn,
prior: torch.Tensor,
length=2048,
tf_board_writer: SummaryWriter = None):
decode_array = prior
for i in Bar('generating').iter(range(min(self.max_seq, length))):
if decode_array.shape[1] >= self.max_seq:
break
_, _, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], decode_array, decode_array)
# result, _ = self.forward(decode_array, lookup_mask=look_ahead_mask)
result, _ = decode_fn(decode_array, look_ahead_mask)
result = self.fc(result)
result = result.softmax(-1)
if tf_board_writer:
tf_board_writer.add_image("logits", result, global_step=i)
u = random.uniform(0, 1)
if u > 1:
result = result[:, -1].argmax(-1).to(torch.int32)
decode_array = torch.cat(
[decode_array, result.unsqueeze(-1)], -1)
else:
pdf = dist.OneHotCategorical(probs=result[:, -1])
result = pdf.sample(1)
result = torch.transpose(result, 1, 0).to(torch.int32)
decode_array = torch.cat((decode_array, result), dim=-1)
del look_ahead_mask
decode_array = decode_array[0]
return decode_array
def forward(self, x, length=None, writer=None):
if self.training or self.eval:
_, _, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, x, config.pad_token)
decoder, w = self.Decoder(x, mask=look_ahead_mask)
fc = self.fc(decoder)
return fc.contiguous(), [weight.contiguous() for weight in w]
else:
return self.generate(self.Decoder, x, length, writer).contiguous()
def forward(self, x, length=None, writer=None):
if self.training or not self.infer:
_, _, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, x, config.pad_token)
decoder, w = self.Decoder(x, mask=look_ahead_mask)
fc = self.fc(decoder)
if self.training:
return fc.contiguous()
else: # still training
return fc.contiguous(), [weight.contiguous() for weight in w]
else:
return self.generate(x, length, None).contiguous().tolist()
def generate_beam(self, prior: list, length=2048, k=8, temperature=0.5):
decode_array = prior
decode_array = tf.constant([decode_array])
seq_probs = [1.0]
for i in range(min(self.max_seq, length)):
if decode_array.shape[1] >= self.max_seq:
break
print('generating... {:.1f}% completed'.format(
(i / min(self.max_seq, length)) * 100),
end="\r")
_, _, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], decode_array, decode_array, self.pad_token)
result = self.call(decode_array,
lookup_mask=look_ahead_mask,
training=False,
eval=False)
result = result[:, -1, :] / (temperature + 1e-6)
probs, result_idx = tf.nn.top_k(result, k)
result_array = []
new_seq_probs = []
for i in range(result_idx.shape[0]):
for j in range(result_idx.shape[1]):
result_unit = tf.concat(
[decode_array[i], [result_idx[i, j]]], -1)
new_seq_probs.append(seq_probs[i] * probs[i, j].numpy())
result_array.append(result_unit.numpy())
seq_probs, idx = tf.nn.top_k(new_seq_probs, k)
decode_array = tf.constant([result_array[i] for i in idx])
# print(seq_probs)
del look_ahead_mask
seq_with_max_prob = np.argmax(seq_probs)
decode_array = decode_array[seq_with_max_prob]
return decode_array.numpy()
def generate(self,
prior: torch.Tensor,
length=2048,
tf_board_writer: SummaryWriter = None):
decode_array = prior
result_array = prior
print(config)
print(length)
for i in Bar('generating').iter(range(length)):
if decode_array.size(1) >= config.threshold_len:
decode_array = decode_array[:, 1:]
_, _, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.size(1), decode_array, decode_array, pad_token=config.pad_token)
# result, _ = self.forward(decode_array, lookup_mask=look_ahead_mask)
# result, _ = decode_fn(decode_array, look_ahead_mask)
result, _ = self.Decoder(decode_array, None)
result = self.fc(result)
result = result.softmax(-1)
if tf_board_writer:
tf_board_writer.add_image("logits", result, global_step=i)
u = 0
if u > 1:
result = result[:, -1].argmax(-1).to(decode_array.dtype)
decode_array = torch.cat((decode_array, result.unsqueeze(-1)),
-1)
else:
pdf = dist.OneHotCategorical(probs=result[:, -1])
print("pdf: " + str(pdf))
print("pdf shape: " + str(pdf.shape))
result = pdf.sample().argmax(-1).unsqueeze(-1)
print("result shape: " + str(result.shape))
# result = torch.transpose(result, 1, 0).to(torch.int32)
decode_array = torch.cat((decode_array, result), dim=-1)
result_array = torch.cat((result_array, result), dim=-1)
del look_ahead_mask
result_array = result_array[0]
return result_array
def sanity_check(self, x, y, mode='v', step=None):
# mode: v -> vector, d -> dict
# x, inp_tar, out_tar = self.__prepare_train_data(x, y)
_, tar_mask, look_ahead_mask = utils.get_masked_with_pad_tensor(
self.max_seq, x, x)
predictions = self.call(x, lookup_mask=look_ahead_mask, training=False)
if mode == 'v':
tf.summary.image('vector', tf.expand_dims(predictions, -1), step)
return predictions
elif mode == 'd':
dic = {}
for row in tf.argmax(predictions, -1).numpy():
for col in row:
try:
dic[str(col)] += 1
except KeyError:
dic[str(col)] = 1
return dic
else:
tf.summary.image('tokens', tf.argmax(predictions, -1), step)
return tf.argmax(predictions, -1)
def generate(self, prior: list, beam=None, length=2048, tf_board=False):
decode_array = prior
decode_array = tf.constant([decode_array])
# TODO: add beam search
if beam is not None:
k = beam
for i in range(min(self.max_seq, length)):
if decode_array.shape[1] >= self.max_seq:
break
if i % 100 == 0:
print('generating... {}% completed'.format(
(i / min(self.max_seq, length)) * 100))
_, _, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], decode_array, decode_array)
result = self.call(decode_array,
lookup_mask=look_ahead_mask,
training=False,
eval=False)
if tf_board:
tf.summary.image('generate_vector',
tf.expand_dims([result[0]], -1), i)
result = result[:, -1, :]
result = tf.reshape(result, (1, -1))
result, result_idx = tf.nn.top_k(result, k)
row = result_idx // par.vocab_size
col = result_idx % par.vocab_size
result_array = []
for r, c in zip(row[0], col[0]):
prev_array = decode_array[r.numpy()]
result_unit = tf.concat([prev_array, [c.numpy()]], -1)
result_array.append(result_unit.numpy())
# result_array.append(tf.concat([decode_array[idx], result[:,idx_idx]], -1))
decode_array = tf.constant(result_array)
del look_ahead_mask
decode_array = decode_array[0]
else:
# for i in Bar('generating').iter(range(min(self.max_seq, length))):
for i in range(min(self.max_seq, length)):
# print(decode_array.shape[1])
if decode_array.shape[1] >= self.max_seq:
break
# if i % 100 == 0:
# print('generating... {}% completed'.format((i/min(self.max_seq, length))*100))
_, _, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], decode_array, decode_array)
result = self.call(decode_array,
lookup_mask=look_ahead_mask,
training=False)
if tf_board:
tf.summary.image('generate_vector',
tf.expand_dims(result, -1), i)
# import sys
# tf.print('[debug out:]', result, sys.stdout )
u = random.uniform(0, 1)
if u > 1:
result = tf.argmax(result[:, -1], -1)
result = tf.cast(result, tf.int32)
decode_array = tf.concat(
[decode_array,
tf.expand_dims(result, -1)], -1)
else:
pdf = tfp.distributions.Categorical(probs=result[:, -1])
result = pdf.sample(1)
result = tf.transpose(result, (1, 0))
result = tf.cast(result, tf.int32)
decode_array = tf.concat([decode_array, result], -1)
# decode_array = tf.concat([decode_array, tf.expand_dims(result[:, -1], 0)], -1)
del look_ahead_mask
decode_array = decode_array[0]
return decode_array.numpy()
def generate(self, prior: list, beam=None, length=2048, tf_board=False):
prior = tf.constant([prior])
decode_array = [par.token_sos]
# TODO: add beam search
if beam is not None:
k = beam
decode_array = tf.constant([decode_array])
for i in range(min(self.max_seq, length)):
# if i % 100 == 0:
# print('generating... {}% completed'.format((i/min(self.max_seq, length))*100))
enc_mask, tar_mask, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], prior, decode_array)
result = self.call(prior,
targets=decode_array,
src_mask=enc_mask,
trg_mask=tar_mask,
lookup_mask=look_ahead_mask,
training=False)
result = result[:, -1, :]
result = tf.reshape(result, (1, -1))
result, result_idx = tf.nn.top_k(result, k)
row = result_idx // par.vocab_size
col = result_idx % par.vocab_size
result_array = []
for r, c in zip(row[0], col[0]):
prev_array = decode_array[r.numpy()]
result_unit = tf.concat([prev_array, [c.numpy()]], -1)
result_array.append(result_unit.numpy())
# result_array.append(tf.concat([decode_array[idx], result[:,idx_idx]], -1))
decode_array = tf.constant(result_array)
del enc_mask
del tar_mask
del look_ahead_mask
decode_array = decode_array[0]
else:
decode_array = tf.constant([decode_array])
# for i in Bar('generating').iter(range(min(self.max_seq, length))):
for i in range(min(self.max_seq, length)):
# if i % 100 == 0:
# print('generating... {}% completed'.format((i/min(self.max_seq, length))*100))
enc_mask, tar_mask, look_ahead_mask = \
utils.get_masked_with_pad_tensor(decode_array.shape[1], prior, decode_array)
result = self.call(prior,
targets=decode_array,
src_mask=enc_mask,
trg_mask=tar_mask,
lookup_mask=look_ahead_mask,
training=False)
result = tf.argmax(result, -1)
result = tf.cast(result, tf.int32)
decode_array = tf.concat(
[decode_array,
tf.expand_dims(result[:, -1], 0)], -1)
del enc_mask
del tar_mask
del look_ahead_mask
return decode_array.numpy()
return seq
def decode(self, token):
return Event.from_int(token)
if __name__ == '__main__':
# import utils
print(tf.executing_eagerly())
src = tf.constant(
[utils.fill_with_placeholder([1, 2, 3, 4], max_len=2048)])
trg = tf.constant(
[utils.fill_with_placeholder([1, 2, 3, 4], max_len=2048)])
src_mask, trg_mask, lookup_mask = utils.get_masked_with_pad_tensor(
2048, src, trg)
print(lookup_mask)
print(src_mask)
mt = MusicTransformer(debug=True,
embedding_dim=par.embedding_dim,
vocab_size=par.vocab_size)
mt.save_weights('my_model.h5', save_format='h5')
mt.load_weights('my_model.h5')
result = mt.generate([27, 186, 43, 213, 115, 131], length=100)
print(result)
from deprecated import sequence
sequence.EventSeq.from_array(
result[0]).to_note_seq().to_midi_file('result.midi')
pass
if __name__ == '__main__':
rga = RelativeGlobalAttention(d=9, h=1)
q = tf.constant(
[[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]],
[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]]],
dtype=tf.float32)
k = tf.constant(
[[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]],
[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]]],
dtype=tf.float32)
import utils
src_mask, trg_mask, look_ahead_mask = utils.get_masked_with_pad_tensor(
q.shape[1], tf.argmax(k, -1), tf.argmax(q, -1))
# print(src_mask.shape, trg_mask.shape, look_ahead_mask.shape)
result = rga([
tf.constant(
[[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]],
[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]]],
dtype=tf.float32),
tf.constant(
[[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]],
[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]]],
dtype=tf.float32),
tf.constant(
[[[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1],
