eval_preiction, weights = single_mt.forward(eval_x) eval_metrics = metric_set(eval_preiction, eval_y) torch.save(single_mt.state_dict(), args.model_dir + '/train-{}.pth'.format(e)) if b == 0: train_summary_writer.add_histogram("target_analysis", batch_y, global_step=e) train_summary_writer.add_histogram("source_analysis", batch_x, global_step=e) for i, weight in enumerate(weights): attn_log_name = "attn/layer-{}".format(i) utils.attention_image_summary(attn_log_name, weight, step=idx, writer=eval_summary_writer) eval_summary_writer.add_scalar('loss', eval_metrics['loss'], global_step=idx) eval_summary_writer.add_scalar('accuracy', eval_metrics['accuracy'], global_step=idx) eval_summary_writer.add_histogram("logits_bucket", eval_metrics['bucket'], global_step=idx) print('\n====================================================') print('Epoch/Batch: {}/{}'.format(e, b)) print('Train >>>> Loss: {:6.6}, Accuracy: {}'.format(
if b == 0: tf.summary.histogram("target_analysis", batch_y, step=e) tf.summary.histogram("source_analysis", batch_x, step=e) tf.summary.scalar('loss', result_metrics[0], step=idx) tf.summary.scalar('accuracy', result_metrics[1], step=idx) with eval_summary_writer.as_default(): if b == 0: mt.sanity_check(eval_x, eval_y, step=e) tf.summary.scalar('loss', eval_result_metrics[0], step=idx) tf.summary.scalar('accuracy', eval_result_metrics[1], step=idx) for i, weight in enumerate(weights): with tf.name_scope("layer_%d" % i): with tf.name_scope("w"): utils.attention_image_summary(weight, step=idx) # for i, weight in enumerate(weights): # with tf.name_scope("layer_%d" % i): # with tf.name_scope("_w0"): # utils.attention_image_summary(weight[0]) # with tf.name_scope("_w1"): # utils.attention_image_summary(weight[1]) idx += 1 print('\n====================================================') print('Epoch/Batch: {}/{}'.format(e, b)) print('Train >>>> Loss: {:6.6}, Accuracy: {}'.format(result_metrics[0], result_metrics[1])) print('Eval >>>> Loss: {:6.6}, Accuracy: {}'.format(eval_result_metrics[0], eval_result_metrics[1]))
def train(num_layers, length, rate, batch, epochs, load_path, save_path, preproc_dir): if rate is None: rate = callback.CustomSchedule(par.embedding_dim) preproc_dir = Path(preproc_dir) model = MusicTransformerDecoder( embedding_dim=256, vocab_size=par.vocab_size, num_layer=num_layers, max_seq=length, dropout=0.2, debug=False, loader_path=load_path, ) model.compile( optimizer=Adam(rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9), loss=callback.transformer_dist_train_loss, ) time = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") train_summary_writer = tf.summary.create_file_writer( f"logs/mt_decoder/{time}/train") eval_summary_writer = tf.summary.create_file_writer( f"logs/mt_decoder/{time}/eval") dataset = Data(preproc_dir) idx = 0 with click.progressbar(length=epochs) as prog: for e in prog: model.reset_metrics() with click.progressbar(length=len(dataset.files) // batch) as prog2: for b in prog2: batch_x, batch_y = dataset.slide_seq2seq_batch( batch, length) loss, acc = model.train_on_batch(batch_x, batch_y) if b % 100 == 0: eval_x, eval_y = dataset.slide_seq2seq_batch( batch, length, "eval") (eloss, eacc), weights = model.evaluate(eval_x, eval_y) if save_path is not None: model.save(save_path) with train_summary_writer.as_default(): if b == 0: tf.summary.histogram("target_analysis", batch_y, step=e) tf.summary.histogram("source_analysis", batch_x, step=e) tf.summary.scalar("loss", loss, step=idx) tf.summary.scalar("accuracy", acc, step=idx) with eval_summary_writer.as_default(): if b == 0: model.sanity_check(eval_x, eval_y, step=e) tf.summary.scalar("loss", eloss, step=idx) tf.summary.scalar("accuracy", eacc, step=idx) for i, weight in enumerate(weights): with tf.name_scope("layer_%d" % i): with tf.name_scope("w"): utils.attention_image_summary(weight, step=idx) print( f"Loss: {loss:6.6} (e: {eloss:6.6}), Accuracy: {acc} (e: {eacc})" ) idx += 1
try: batch_x, batch_y = dataset.seq2seq_batch(batch_size, max_seq) except: continue result_metrics = mt.train_on_batch(batch_x, batch_y) if b % 100 == 0: eval_x, eval_y = dataset.seq2seq_batch(batch_size, max_seq, 'eval') eval_result_metrics, weights = mt.evaluate(eval_x, eval_y) mt.save(save_path) with train_summary_writer.as_default(): tf.summary.scalar('loss', result_metrics[0], step=idx) tf.summary.scalar('accuracy', result_metrics[1], step=idx) for i, weight in enumerate(weights): with tf.name_scope("layer_%d" % i): with tf.name_scope("_w0"): utils.attention_image_summary(weight[0]) with tf.name_scope("_w1"): utils.attention_image_summary(weight[1]) with eval_summary_writer.as_default(): tf.summary.scalar('loss', eval_result_metrics[0], step=idx) tf.summary.scalar('accuracy', eval_result_metrics[1], step=idx) idx += 1 print('\n====================================================') print('Epoch/Batch: {}/{}'.format(e, b)) print('Train >>>> Loss: {:6.6}, Accuracy: {}'.format( result_metrics[0], result_metrics[1])) print('Eval >>>> Loss: {:6.6}, Accuracy: {}'.format( eval_result_metrics[0], eval_result_metrics[1]))
def train(input_path, save_path, l_r=None, batch_size=2, max_seq=1024, epochs=100, load_path=None, num_layer=6, log_dir='/pfs/out/logs'): # load data dataset = Data(input_path) print('dataset', dataset) # load model learning_rate = callback.CustomSchedule(par.embedding_dim) if l_r is None else l_r opt = Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9) # define model mt = MusicTransformerDecoder( embedding_dim=256, vocab_size=par.vocab_size, num_layer=num_layer, max_seq=max_seq, dropout=0.2, debug=False, loader_path=load_path) mt.compile(optimizer=opt, loss=callback.transformer_dist_train_loss) # define tensorboard writer current_time = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') train_log_dir = '{log_dir}/{time}/train'.format(log_dir=log_dir, time=current_time) eval_log_dir = '{log_dir}/{time}/eval'.format(log_dir=log_dir, time=current_time) train_summary_writer = tf.summary.create_file_writer(train_log_dir) eval_summary_writer = tf.summary.create_file_writer(eval_log_dir) # Train Start idx = 0 batchings = len(dataset.files) // batch_size how_often_to_print = 50 for e in tqdm(range(epochs), desc='epochs'): mt.reset_metrics() for b in tqdm(range(batchings), desc='batches'): try: batch_x, batch_y = dataset.slide_seq2seq_batch(batch_size, max_seq) except: continue result_metrics = mt.train_on_batch(batch_x, batch_y) if b % how_often_to_print == 0: eval_x, eval_y = dataset.slide_seq2seq_batch(batch_size, max_seq, 'eval') eval_result_metrics, weights = mt.evaluate(eval_x, eval_y) mt.save(save_path) with train_summary_writer.as_default(): if b == 0: tf.summary.histogram("target_analysis", batch_y, step=e) tf.summary.histogram("source_analysis", batch_x, step=e) tf.summary.scalar('loss', result_metrics[0], step=idx) tf.summary.scalar('accuracy', result_metrics[1], step=idx) with eval_summary_writer.as_default(): if b == 0: mt.sanity_check(eval_x, eval_y, step=e) tf.summary.scalar('loss', eval_result_metrics[0], step=idx) tf.summary.scalar('accuracy', eval_result_metrics[1], step=idx) for i, weight in enumerate(weights): with tf.name_scope("layer_%d" % i): with tf.name_scope("w"): utils.attention_image_summary(weight, step=idx) # for i, weight in enumerate(weights): # with tf.name_scope("layer_%d" % i): # with tf.name_scope("_w0"): # utils.attention_image_summary(weight[0]) # with tf.name_scope("_w1"): # utils.attention_image_summary(weight[1]) idx += 1 print('\n====================================================') print('Epoch/Batch: {}/{}'.format(e, b)) print('Train >>>> Loss: {:6.6}, Accuracy: {}'.format(result_metrics[0], result_metrics[1])) print('Eval >>>> Loss: {:6.6}, Accuracy: {}'.format(eval_result_metrics[0], eval_result_metrics[1]))