def train(self,
feature_folder,
semi_feature_folder=None,
model_name=None,
input_model_path=None,
vocal_settings=None):
"""Model training.
Train a new model or continue to train on a previously trained model.
Parameters
----------
feature_folder: Path
Path to the folder containing generated feature.
semi_feature_folder: Path
If specified, semi-supervise learning will be leveraged, and the feature
files contained in this folder will be used as unsupervised data.
model_name: str
The name for storing the trained model. If not given, will default to the
current timesamp.
input_model_path: Path
Continue to train on the pre-trained model by specifying the path.
vocal_settings: VocalSettings
The configuration instance that holds all relative settings for
the life-cycle of building a model.
"""
settings = self._validate_and_get_settings(vocal_settings)
if input_model_path is not None:
logger.info("Continue to train on model: %s", input_model_path)
model, prev_set = self._load_model(input_model_path)
settings.model.save_path = prev_set.model.save_path
logger.info("Constructing dataset instance")
split = settings.training.steps / (settings.training.steps +
settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(
feature_folder, split=split)
output_types = (tf.float32, tf.float32)
output_shapes = ((settings.training.context_length * 2 + 1, 174, 9),
(19, 6)) # noqa: E226
train_dataset = VocalDatasetLoader(
ctx_len=settings.training.context_length,
feature_files=train_feat_files,
num_samples=settings.training.epoch * settings.training.batch_size * settings.training.steps
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
val_dataset = VocalDatasetLoader(
ctx_len=settings.training.context_length,
feature_files=val_feat_files,
num_samples=settings.training.epoch * settings.training.val_batch_size * settings.training.val_steps
) \
.get_dataset(settings.training.val_batch_size, output_types=output_types, output_shapes=output_shapes)
if semi_feature_folder is not None:
# Semi-supervise learning dataset.
feat_files = glob.glob(f"{semi_feature_folder}/*.hdf")
semi_dataset = VocalDatasetLoader(
ctx_len=settings.training.context_length,
feature_files=feat_files,
num_samples=settings.training.epoch * settings.training.batch_size * settings.training.steps
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
train_dataset = tf.data.Dataset.zip((train_dataset, semi_dataset))
if input_model_path is None:
logger.info("Constructing new model")
model = self.get_model(settings)
# Notice: the original implementation uses AdamW as the optimizer, which is also viable through
# tensorflow_addons.optimizers.AdamW. However we found that by using AdamW, the model would fail
# to converge, and instead the training loss will get higher and higher.
optimizer = tf.keras.optimizers.Adam(
learning_rate=settings.training.init_learning_rate)
model.compile(optimizer=optimizer,
loss='bce',
metrics=['accuracy', 'binary_accuracy'])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(),
jpath(model_save_path, "configurations.yaml"))
logger.info("Model output to: %s", model_save_path)
logger.info("Constructing callbacks")
callbacks = [
tf.keras.callbacks.EarlyStopping(
patience=settings.training.early_stop, monitor="val_loss"),
tf.keras.callbacks.ModelCheckpoint(jpath(model_save_path,
"weights"),
save_weights_only=True,
monitor="val_loss")
]
logger.info("Callback list: %s", callbacks)
logger.info("Start training")
history = model.fit(train_dataset,
validation_data=val_dataset,
epochs=settings.training.epoch,
steps_per_epoch=settings.training.steps,
validation_steps=settings.training.val_steps,
callbacks=callbacks,
use_multiprocessing=True,
workers=8)
return model_save_path, history
def train(self,
feature_folder,
model_name=None,
input_model_path=None,
music_settings=None):
"""Model training.
Train the model from scratch or continue training given a model checkpoint.
Parameters
----------
feature_folder: Path
Path to the generated feature.
model_name: str
The name of the trained model. If not given, will default to the
current timestamp.
input_model_path: Path
Specify the path to the model checkpoint in order to fine-tune
the model.
music_settings: MusicSettings
The configuration that holds all relative settings for
the life-cycle of model building.
"""
settings = self._validate_and_get_settings(music_settings)
if input_model_path is not None:
logger.info("Continue to train on model: %s", input_model_path)
model, prev_set = self._load_model(
input_model_path, custom_objects=self.custom_objects)
settings.training.timesteps = prev_set.training.timesteps
settings.training.label_type = prev_set.training.label_type
settings.training.channels = prev_set.training.channels
settings.model.save_path = prev_set.model.save_path
settings.transcription_mode = prev_set.transcription_mode
logger.info("Using label type: %s", settings.training.label_type)
l_type = LabelType(settings.training.label_type)
settings.transcription_mode = self.label_trans_mode_mapping[
settings.training.label_type]
logger.info("Constructing dataset instance")
split = settings.training.steps / (settings.training.steps +
settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(
feature_folder, split=split)
output_types = (tf.float32, tf.float32)
train_dataset = MusicDatasetLoader(
l_type.get_conversion_func(),
feature_files=train_feat_files,
num_samples=settings.training.batch_size * settings.training.steps,
timesteps=settings.training.timesteps,
channels=[FEATURE_NAME_TO_NUMBER[ch_name] for ch_name in settings.training.channels],
feature_num=settings.training.feature_num
) \
.get_dataset(settings.training.batch_size, output_types=output_types)
val_dataset = MusicDatasetLoader(
l_type.get_conversion_func(),
feature_files=val_feat_files,
num_samples=settings.training.val_batch_size * settings.training.val_steps,
timesteps=settings.training.timesteps,
channels=[FEATURE_NAME_TO_NUMBER[ch_name] for ch_name in settings.training.channels],
feature_num=settings.training.feature_num
) \
.get_dataset(settings.training.val_batch_size, output_types=output_types)
if input_model_path is None:
logger.info("Creating new model with type: %s",
settings.model.model_type)
model_func = {
"aspp": semantic_segmentation,
"attn": semantic_segmentation_attn
}[settings.model.model_type]
model = model_func(timesteps=settings.training.timesteps,
out_class=l_type.get_out_classes(),
ch_num=len(settings.training.channels))
logger.info("Compiling model with loss function type: %s",
settings.training.loss_function)
loss_func = {
"smooth":
lambda y, x: smooth_loss(y, x, total_chs=l_type.get_out_classes()),
"focal":
focal_loss,
"bce":
tf.keras.losses.BinaryCrossentropy()
}[settings.training.loss_function]
model.compile(optimizer="adam", loss=loss_func, metrics=['accuracy'])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(),
jpath(model_save_path, "configurations.yaml"))
write_yaml(model.to_yaml(),
jpath(model_save_path, "arch.yaml"),
dump=False)
logger.info("Model output to: %s", model_save_path)
logger.info("Constructing callbacks")
callbacks = [
EarlyStopping(patience=settings.training.early_stop),
ModelCheckpoint(model_save_path, save_weights_only=True)
]
logger.info("Callback list: %s", callbacks)
logger.info("Start training")
history = train_epochs(model,
train_dataset,
validate_dataset=val_dataset,
epochs=settings.training.epoch,
steps=settings.training.steps,
val_steps=settings.training.val_steps,
callbacks=callbacks)
return model_save_path, history
def train(self, feature_folder, model_name=None, input_model_path=None, beat_settings=None):
"""Model training.
Train the model from scratch or continue training given a model checkpoint.
Parameters
----------
feature_folder: Path
Path to the generated feature.
model_name: str
The name of the trained model. If not given, will default to the
current timestamp.
input_model_path: Path
Specify the path to the model checkpoint in order to fine-tune
the model.
beat_settings: BeatSettings
The configuration that holds all relative settings for
the life-cycle of model building.
"""
settings = self._validate_and_get_settings(beat_settings)
if input_model_path is not None:
logger.info("Continue to train on model: %s", input_model_path)
model, prev_set = self._load_model(input_model_path)
settings.model.from_json(prev_set.model.to_json())
settings.feature.time_unit = prev_set.feature.time_unit
logger.info("Constructing dataset instance")
split = settings.training.steps / (settings.training.steps + settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(feature_folder, split=split)
output_types = (tf.float32, tf.float32)
output_shapes = ((settings.model.timesteps, 178), (settings.model.timesteps, 2))
train_dataset = BeatDatasetLoader(
feature_files=train_feat_files,
num_samples=settings.training.epoch * settings.training.batch_size * settings.training.steps,
slice_hop=settings.model.timesteps // 2
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
val_dataset = BeatDatasetLoader(
feature_files=val_feat_files,
num_samples=settings.training.epoch * settings.training.val_batch_size * settings.training.val_steps,
slice_hop=settings.model.timesteps // 2
) \
.get_dataset(settings.training.val_batch_size, output_types=output_types, output_shapes=output_shapes)
if input_model_path is None:
logger.info("Constructing new %s model for training.", settings.model.model_type)
model_func = {
"blstm": self._construct_blstm_model,
"blstm_attn": self._construct_blstm_attn_model
}[settings.model.model_type]
model = model_func(settings)
logger.info("Compiling model")
optimizer = tf.keras.optimizers.Adam(learning_rate=settings.training.init_learning_rate)
loss = lambda y, x: weighted_binary_crossentropy(y, x, down_beat_weight=settings.training.down_beat_weight)
model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(), jpath(model_save_path, "configurations.yaml"))
write_yaml(model.to_yaml(), jpath(model_save_path, "arch.yaml"), dump=False)
logger.info("Model output to: %s", model_save_path)
logger.info("Constructing callbacks")
callbacks = [
tf.keras.callbacks.EarlyStopping(
patience=settings.training.early_stop, monitor="val_loss", restore_best_weights=False
),
tf.keras.callbacks.ModelCheckpoint(
jpath(model_save_path, "weights.h5"), save_weights_only=True, monitor="val_loss"
)
]
logger.info("Callback list: %s", callbacks)
logger.info("Start training")
history = model.fit(
train_dataset,
validation_data=val_dataset,
epochs=settings.training.epoch,
steps_per_epoch=settings.training.steps,
validation_steps=settings.training.val_steps,
callbacks=callbacks,
use_multiprocessing=True,
workers=8
)
return model_save_path, history
def train(self,
feature_folder,
model_name=None,
input_model_path=None,
drum_settings=None):
"""Model training.
Train a new model or continue to train on a previously trained model.
Parameters
----------
feature_folder: Path
Path to the folder containing generated feature.
model_name: str
The name for storing the trained model. If not given, will default to the
current timesamp.
input_model_path: Path
Continue to train on the pre-trained model by specifying the path.
drum_settings: DrumSettings
The configuration instance that holds all relative settings for
the life-cycle of building a model.
"""
settings = self._validate_and_get_settings(drum_settings)
if input_model_path is not None:
logger.info("Continue to train on model: %s", input_model_path)
model, prev_set = self._load_model(
input_model_path, custom_objects=self.custom_objects)
settings.model.save_path = prev_set.model.save_path
settings.training.init_learninig_rate = prev_set.training.init_learning_rate
settings.training.res_block_num = prev_set.training.res_block_num
logger.info("Constructing dataset instance")
split = settings.training.steps / (settings.training.steps +
settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(
feature_folder, split=split)
output_types = (tf.float32, tf.float32)
output_shapes = ([120, 120, 4], [4, 13])
train_dataset = PopDatasetLoader(
feature_files=train_feat_files,
num_samples=settings.training.epoch * settings.training.batch_size * settings.training.steps
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
val_dataset = PopDatasetLoader(
feature_files=val_feat_files,
num_samples=settings.training.epoch * settings.training.val_batch_size * settings.training.val_steps
) \
.get_dataset(settings.training.val_batch_size, output_types=output_types, output_shapes=output_shapes)
if input_model_path is None:
logger.info("Constructing new model")
model = drum_model(
out_classes=13,
mini_beat_per_seg=settings.feature.mini_beat_per_segment,
res_block_num=settings.training.res_block_num)
optimizer = tf.keras.optimizers.Adam(
learning_rate=settings.training.init_learning_rate)
model.compile(optimizer=optimizer,
loss=loss_func,
metrics=["accuracy"])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(),
jpath(model_save_path, "configurations.yaml"))
write_yaml(model.to_yaml(),
jpath(model_save_path, "arch.yaml"),
dump=False)
logger.info("Model output to: %s", model_save_path)
logger.info("Constructing callbacks")
callbacks = [
tf.keras.callbacks.EarlyStopping(
patience=settings.training.early_stop, monitor="val_loss"),
tf.keras.callbacks.ModelCheckpoint(jpath(model_save_path,
"weights.h5"),
save_weights_only=True)
]
logger.info("Callback list: %s", callbacks)
logger.info("Start training")
history = model.fit(train_dataset,
validation_data=val_dataset,
epochs=settings.training.epoch,
steps_per_epoch=settings.training.steps,
validation_steps=settings.training.val_steps,
callbacks=callbacks,
use_multiprocessing=True,
workers=8)
return model_save_path, history
def train(self,
feature_folder,
model_name=None,
input_model_path=None,
chord_settings=None):
"""Model training.
Train a new music model or continue to train on a pre-trained model.
Parameters
----------
feature_folder: Path
Path to the generated feature.
model_name: str
The name of the trained model. If not given, will default to the
current timestamp.
input_model_path: Path
Specify the path to the pre-trained model if you want to continue
to fine-tune on the model.
chord_settings: ChordSettings
The configuration instance that holds all relative settings for
the life-cycle of building a model.
"""
settings = self._validate_and_get_settings(chord_settings)
if input_model_path is not None:
logger.info("Continue to train one model: %s", input_model_path)
model, _ = self._load_model(input_model_path)
split = settings.training.steps / (settings.training.steps +
settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(
feature_folder, split=split)
output_types = (tf.float32, (tf.int32, tf.int32))
output_shapes = ([
settings.feature.num_steps, settings.feature.segment_width * 24
], ([settings.feature.num_steps], [settings.feature.num_steps]))
train_dataset = McGillDatasetLoader(
feature_files=train_feat_files,
num_samples=settings.training.epoch * settings.training.batch_size * settings.training.steps
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
val_dataset = McGillDatasetLoader(
feature_files=val_feat_files,
num_samples=settings.training.epoch * settings.training.val_batch_size * settings.training.val_steps
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
if input_model_path is None:
logger.info("Constructing new model")
model = self.get_model(settings)
learninig_rate = tf.keras.optimizers.schedules.ExponentialDecay(
settings.training.init_learning_rate,
decay_steps=settings.training.steps,
decay_rate=settings.training.learning_rate_decay,
staircase=True)
optimizer = tf.keras.optimizers.Adam(learning_rate=learninig_rate,
clipvalue=1)
model.compile(optimizer=optimizer,
loss=chord_loss_func,
metrics=["accuracy"])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(),
jpath(model_save_path, "configurations.yaml"))
logger.info("Model output to: %s", model_save_path)
callbacks = [
tf.keras.callbacks.EarlyStopping(
patience=settings.training.early_stop, monitor="val_loss"),
tf.keras.callbacks.ModelCheckpoint(jpath(model_save_path,
"weights"),
save_weights_only=True,
monitor="val_loss"),
ReduceSlope()
]
history = model.fit(train_dataset,
validation_data=val_dataset,
epochs=settings.training.epoch,
steps_per_epoch=settings.training.steps,
validation_steps=settings.training.val_steps,
callbacks=callbacks)
return history
def train(self,
feature_folder,
model_name=None,
input_model_path=None,
patch_cnn_settings=None):
"""Model training.
Train the model from scratch or continue training given a model checkpoint.
Parameters
----------
feature_folder: Path
Path to the generated feature.
model_name: str
The name of the trained model. If not given, will default to the
current timestamp.
input_model_path: Path
Specify the path to the model checkpoint in order to fine-tune
the model.
patch_cnn_settings: VocalContourSettings
The configuration that holds all relative settings for
the life-cycle of model building.
"""
settings = self._validate_and_get_settings(patch_cnn_settings)
if input_model_path is not None:
logger.info("Continue to train on model: %s", input_model_path)
model, prev_set = self._load_model(
input_model_path, custom_objects=self.custom_objects)
settings.feature.patch_size = prev_set.feature.patch_size
logger.info("Constructing dataset instance")
split = settings.training.steps / (settings.training.steps +
settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(
feature_folder, split=split)
output_types = (tf.float32, tf.float32)
output_shapes = ((settings.feature.patch_size,
settings.feature.patch_size, 1), (2))
train_dataset = PatchCNNDatasetLoader(
feature_files=train_feat_files,
num_samples=settings.training.epoch * settings.training.batch_size * settings.training.steps
) \
.get_dataset(settings.training.batch_size, output_types=output_types, output_shapes=output_shapes)
val_dataset = PatchCNNDatasetLoader(
feature_files=val_feat_files,
num_samples=settings.training.epoch * settings.training.val_batch_size * settings.training.val_steps
) \
.get_dataset(settings.training.val_batch_size, output_types=output_types, output_shapes=output_shapes)
if input_model_path is None:
logger.info("Constructing new model")
model = patch_cnn_model(patch_size=settings.feature.patch_size)
logger.info("Compiling model")
optimizer = tf.keras.optimizers.Adam(
learning_rate=settings.training.init_learning_rate)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(),
jpath(model_save_path, "configurations.yaml"))
write_yaml(model.to_yaml(),
jpath(model_save_path, "arch.yaml"),
dump=False)
logger.info("Model output to: %s", model_save_path)
logger.info("Constrcuting callbacks")
callbacks = [
tf.keras.callbacks.EarlyStopping(
patience=settings.training.early_stop),
tf.keras.callbacks.ModelCheckpoint(jpath(model_save_path,
"weights.h5"),
save_weights_only=True)
]
logger.info("Callback list: %s", callbacks)
logger.info("Start training")
history = model.fit(train_dataset,
validation_data=val_dataset,
epochs=settings.training.epoch,
steps_per_epoch=settings.training.steps,
validation_steps=settings.training.val_steps,
callbacks=callbacks,
use_multiprocessing=True,
workers=8)
return model_save_path, history
def train(self,
feature_folder,
model_name=None,
input_model_path=None,
vocalcontour_settings=None):
"""Model training.
Train the model from scratch or continue training given a model checkpoint.
Parameters
----------
feature_folder: Path
Path to the generated feature.
model_name: str
The name of the trained model. If not given, will default to the
current timestamp.
input_model_path: Path
Specify the path to the model checkpoint in order to fine-tune
the model.
vocalcontour_settings: VocalContourSettings
The configuration that holds all relative settings for
the life-cycle of model building.
"""
settings = self._validate_and_get_settings(vocalcontour_settings)
if input_model_path is not None:
logger.info("Continue to train one model: %s", input_model_path)
model, prev_set = self._load_model(input_model_path)
settings.training.timesteps = prev_set.training.timesteps
settings.model.save_path = prev_set.model.save_path
logger.info("Constructing dataset instance")
split = settings.training.steps / (settings.training.steps +
settings.training.val_steps)
train_feat_files, val_feat_files = get_train_val_feat_file_list(
feature_folder, split=split)
output_types = (tf.float32, tf.float32)
train_dataset = VocalContourDatasetLoader(
feature_files=train_feat_files,
num_samples=settings.training.batch_size * settings.training.steps,
timesteps=settings.training.timesteps).get_dataset(
settings.training.batch_size, output_types=output_types)
val_dataset = VocalContourDatasetLoader(
feature_files=val_feat_files,
num_samples=settings.training.val_batch_size *
settings.training.val_steps,
timesteps=settings.training.timesteps).get_dataset(
settings.training.val_batch_size, output_types=output_types)
if input_model_path is None:
logger.info("Constructing new model")
# NOTE: The default value of dropout rate for ConvBlock is different
# in VocalSeg which is 0.2.
model = semantic_segmentation(
multi_grid_layer_n=1,
feature_num=384,
ch_num=1,
timesteps=settings.training.timesteps)
model.compile(optimizer="adam", loss=focal_loss, metrics=['accuracy'])
logger.info("Resolving model output path")
if model_name is None:
model_name = str(datetime.now()).replace(" ", "_")
if not model_name.startswith(settings.model.save_prefix):
model_name = settings.model.save_prefix + "_" + model_name
model_save_path = jpath(settings.model.save_path, model_name)
ensure_path_exists(model_save_path)
write_yaml(settings.to_json(),
jpath(model_save_path, "configurations.yaml"))
write_yaml(model.to_yaml(),
jpath(model_save_path, "arch.yaml"),
dump=False)
logger.info("Model output to: %s", model_save_path)
logger.info("Constructing callbacks")
callbacks = [
EarlyStopping(patience=settings.training.early_stop),
ModelCheckpoint(model_save_path, save_weights_only=True)
]
logger.info("Callback list: %s", callbacks)
logger.info("Start training")
history = train_epochs(model,
train_dataset,
validate_dataset=val_dataset,
epochs=settings.training.epoch,
steps=settings.training.steps,
val_steps=settings.training.val_steps,
callbacks=callbacks)
return model_save_path, history