def build_model(args, i_cv):
args.net = ARCHI(n_in=29, n_out=2, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
model = get_model(args, Regressor)
model.base_name = CALIB
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
def build_model(args, i_cv):
args.net = ARCHI(n_in=3, n_out=args.n_bins)
args.optimizer = get_optimizer(args)
args.criterion = S3DLoss()
model = get_model(args, Inferno)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
def main():
parser = argparse.ArgumentParser(description='Proxyless-NAS augment')
parser.add_argument('-n',
'--name',
type=str,
required=True,
help="name of the model")
parser.add_argument('-c',
'--config',
type=str,
default='./config/default.yaml',
help="yaml config file")
parser.add_argument('-p',
'--chkpt',
type=str,
default=None,
help="path of checkpoint pt file")
parser.add_argument('-d',
'--device',
type=str,
default="all",
help="override device ids")
parser.add_argument('-g',
'--genotype',
type=str,
default=None,
help="override genotype file")
args = parser.parse_args()
hp = HParam(args.config)
pt_path = os.path.join('.', hp.log.chkpt_dir)
out_dir = os.path.join(pt_path, args.name)
os.makedirs(out_dir, exist_ok=True)
log_dir = os.path.join('.', hp.log.log_dir)
log_dir = os.path.join(log_dir, args.name)
os.makedirs(log_dir, exist_ok=True)
logger = utils.get_logger(log_dir, args.name)
if utils.check_config(hp, args.name):
raise Exception("Config error.")
writer = utils.get_writer(log_dir, hp.log.writer)
dev, dev_list = utils.init_device(hp.device, args.device)
trn_loader = load_data(hp.augment.data, validation=False)
val_loader = load_data(hp.augment.data, validation=True)
gt.set_primitives(hp.genotypes)
# load genotype
genotype = utils.get_genotype(hp.augment, args.genotype)
model, arch = get_model(hp.model, dev, dev_list, genotype)
augment(out_dir, args.chkpt, trn_loader, val_loader, model, writer, logger,
dev, hp.augment)
def build_model(args, i_cv):
args.net = ARCHI(n_in=3, n_out=2, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
model = get_model(args, NeuralNetClassifier)
model.base_name = "DataAugmentation"
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
def build_model(args, i_cv):
args.net = ARCHI(n_in=29, n_out=N_BINS, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
args.criterion = HiggsLoss()
model = get_model(args, Inferno)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
def main():
# BASIC SETUP
logger = set_logger()
args = REG_parse_args(
main_description="Training launcher for Regressor on S3D2 benchmark")
logger.info(args)
flush(logger)
# INFO
args.net = AR5R5E(n_in=3, n_out=2, n_extra=2)
args.optimizer = get_optimizer(args)
model = get_model(args, Regressor)
model.set_info(DATA_NAME, BENCHMARK_NAME, -1)
pb_config = S3D2Config()
# RUN
results = [run(args, i_cv) for i_cv in range(N_ITER)]
results = pd.concat(results, ignore_index=True)
results.to_csv(os.path.join(model.results_directory, 'results.csv'))
# EVALUATION
eval_table = evaluate_estimator(pb_config.INTEREST_PARAM_NAME, results)
print_line()
print_line()
print(eval_table)
print_line()
print_line()
eval_table.to_csv(os.path.join(model.results_directory, 'evaluation.csv'))
gather_images(model.results_directory)
def build_model(args, i_cv):
args.net = ARCHI(n_in=29, n_out=2, n_unit=args.n_unit)
args.adv_net = ARCHI(n_in=2, n_out=2, n_unit=args.n_unit)
args.net_optimizer = get_optimizer(args)
args.adv_optimizer = get_optimizer(args, args.adv_net)
args.net_criterion = WeightedCrossEntropyLoss()
args.adv_criterion = WeightedGaussEntropyLoss()
model = get_model(args, PivotClassifier)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
def model_call(text,
speaker_id: int,
restore_step: int,
mode,
pro_path,
model_path,
train_path,
pitch_control=1.0,
energy_control=1.0,
duration_control=1.0):
# Check source texts
source = None
if mode == "batch":
assert source is not None and text is None
if mode == "single":
assert source is None and text is not None
# Read config
preprocess_config = yaml.load(open(pro_path, "r"), Loader=yaml.FullLoader)
model_config = yaml.load(open(model_path, "r"), Loader=yaml.FullLoader)
train_config = yaml.load(open(train_path, "r"), Loader=yaml.FullLoader)
configs = (preprocess_config, model_config, train_config)
# Get model
model = get_model(restore_step, configs, device, train=False)
# Load vocoder
vocoder = get_vocoder(model_config, device)
# Preprocess texts
if mode == "batch":
# Get dataset
dataset = TextDataset(source, preprocess_config)
batchs = DataLoader(
dataset,
batch_size=8,
collate_fn=dataset.collate_fn,
)
if mode == "single":
ids = raw_texts = [text[:100]]
speakers = np.array([speaker_id])
if preprocess_config["preprocessing"]["text"]["language"] == "en":
texts = np.array([preprocess_english(text, preprocess_config)])
elif preprocess_config["preprocessing"]["text"]["language"] == "zh":
texts = np.array([preprocess_mandarin(text, preprocess_config)])
text_lens = np.array([len(texts[0])])
batchs = [(ids, raw_texts, speakers, texts, text_lens, max(text_lens))]
control_values = pitch_control, energy_control, duration_control
synthesize(model, restore_step, configs, vocoder, batchs, control_values)
def load_calib_rescale():
args = lambda: None
args.n_unit = 80
args.optimizer_name = "adam"
args.beta1 = 0.5
args.beta2 = 0.9
args.learning_rate = 1e-4
args.n_samples = 1000
args.n_steps = 1000
args.batch_size = 20
args.net = CALIB_ARCHI(n_in=1, n_out=2, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
model = get_model(args, Regressor)
model.base_name = CALIB_RESCALE
model.set_info(DATA_NAME, BENCHMARK_NAME, 0)
model.load(model.model_path)
return model
def load_calib_lam(DATA_NAME, BENCHMARK_NAME):
from model.regressor import Regressor
from archi.reducer import A1AR8MR8L1 as CALIB_ARCHI
args = lambda: None
args.n_unit = 200
args.optimizer_name = "adam"
args.beta1 = 0.5
args.beta2 = 0.9
args.learning_rate = 1e-4
args.n_samples = 1000
args.n_steps = 2000
args.batch_size = 20
args.net = CALIB_ARCHI(n_in=3, n_out=2, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
model = get_model(args, Regressor)
model.base_name = "Calib_lam"
model.set_info(DATA_NAME, BENCHMARK_NAME, 0)
model.load(model.model_path)
return model
def model_init(restore_step: int,
pro_path,
model_path,
train_path,
pitch_control=1.0,
energy_control=1.0,
duration_control=1.0):
# Read config
preprocess_config = yaml.load(open(pro_path, "r"), Loader=yaml.FullLoader)
model_config = yaml.load(open(model_path, "r"), Loader=yaml.FullLoader)
train_config = yaml.load(open(train_path, "r"), Loader=yaml.FullLoader)
configs = (preprocess_config, model_config, train_config)
# Get model
model = get_model(restore_step, configs, device, train=False)
# Load vocoder
vocoder = get_vocoder(model_config, device)
control_values = pitch_control, energy_control, duration_control
return model, restore_step, configs, vocoder, control_values
def run(config):
pl.seed_everything(config.seed)
if config.logger:
from pytorch_lightning.loggers import WandbLogger
name = f"{config.model_name}-{config.backbone.name}-{config.dataset.src_task}-{config.dataset.tgt_task}"
logger = WandbLogger(
project=f"{config.project}",
name=name,
)
else:
logger = pl.loggers.TestTubeLogger("output", name=f"{config.project}")
logger.log_hyperparams(config)
datamodule = get_datamodule(config.dataset,
batch_size=config.training.batch_size)
model = get_model(config, len(datamodule.CLASSES))
trainer = pl.Trainer(
precision=16,
auto_lr_find=True if config.lr_finder else None,
deterministic=True,
check_val_every_n_epoch=1,
gpus=config.gpus,
logger=logger,
max_epochs=config.training.epoch,
weights_summary="top",
)
if config.lr_finder:
lr_finder = trainer.tuner.lr_find(model,
min_lr=1e-8,
max_lr=1e-1,
num_training=100)
model.hparams.lr = lr_finder.suggestion()
print(model.hparams.lr)
else:
trainer.fit(model, datamodule=datamodule)
trainer.test()
def main():
# BASIC SETUP
logger = set_logger()
args = GB_parse_args(main_description="Training launcher for Gradient boosting on AP1 benchmark")
logger.info(args)
flush(logger)
# INFO
model = get_model(args, GradientBoostingModel)
model.set_info(BENCHMARK_NAME, -1)
pb_config = AP1Config()
# RUN
results = [run(args, i_cv) for i_cv in range(N_ITER)]
results = pd.concat(results, ignore_index=True)
results.to_csv(os.path.join(model.directory, 'results.csv'))
# EVALUATION
eval_table = evaluate_estimator(pb_config.INTEREST_PARAM_NAME, results)
print_line()
print_line()
print(eval_table)
print_line()
print_line()
eval_table.to_csv(os.path.join(model.directory, 'evaluation.csv'))
gather_images(model.directory)
parser = argparse.ArgumentParser()
parser.add_argument("--restore_step", type=int, default=30000)
parser.add_argument(
"-p",
"--preprocess_config",
type=str,
required=True,
help="path to preprocess.yaml",
)
parser.add_argument(
"-m", "--model_config", type=str, required=True, help="path to model.yaml"
)
parser.add_argument(
"-t", "--train_config", type=str, required=True, help="path to train.yaml"
)
args = parser.parse_args()
# Read Config
preprocess_config = yaml.load(
open(args.preprocess_config, "r"), Loader=yaml.FullLoader
)
model_config = yaml.load(open(args.model_config, "r"), Loader=yaml.FullLoader)
train_config = yaml.load(open(args.train_config, "r"), Loader=yaml.FullLoader)
configs = (preprocess_config, model_config, train_config)
# Get model
model = get_model(args, configs, device, train=False).to(device)
message = evaluate(model, args.restore_step, configs)
print(message)
def run(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
result_table = []
# LOAD/GENERATE DATA
logger.info('Set up data generator')
pb_config = S3D2Config()
seed = config.SEED + i_cv * 5
train_generator = S3D2(seed)
valid_generator = S3D2(seed + 1)
test_generator = S3D2(seed + 2)
# SET MODEL
logger.info('Set up rergessor')
args.net = AR5R5E(n_in=3, n_out=2, n_extra=2)
args.optimizer = get_optimizer(args)
model = get_model(args, Regressor)
model.set_info(BENCHMARK_NAME, i_cv)
model.param_generator = param_generator
flush(logger)
# TRAINING / LOADING
if not args.retrain:
try:
logger.info('loading from {}'.format(model.model_path))
model.load(model.model_path)
except Exception as e:
logger.warning(e)
args.retrain = True
if args.retrain:
logger.info('Training {}'.format(model.get_name()))
model.fit(train_generator)
logger.info('Training DONE')
# SAVE MODEL
save_model(model)
# CHECK TRAINING
logger.info('Plot losses')
plot_REG_losses(model)
plot_REG_log_mse(model)
result_row['loss'] = model.losses[-1]
result_row['mse_loss'] = model.mse_losses[-1]
# MEASUREMENT
for mu in pb_config.TRUE_MU_RANGE:
pb_config.TRUE_MU = mu
logger.info('Generate testing data')
test_generator.reset()
X_test, y_test, w_test = test_generator.generate(
# pb_config.TRUE_R,
# pb_config.TRUE_LAMBDA,
pb_config.CALIBRATED_R,
pb_config.CALIBRATED_LAMBDA,
pb_config.TRUE_MU,
n_samples=pb_config.N_TESTING_SAMPLES)
p_test = np.array(
(pb_config.CALIBRATED_R, pb_config.CALIBRATED_LAMBDA))
pred, sigma = model.predict(X_test, w_test, p_test)
name = pb_config.INTEREST_PARAM_NAME
result_row[name] = pred
result_row[name + _ERROR] = sigma
result_row[name + _TRUTH] = pb_config.TRUE_MU
logger.info('{} =vs= {} +/- {}'.format(pb_config.TRUE_MU, pred, sigma))
result_table.append(result_row.copy())
result_table = pd.DataFrame(result_table)
logger.info('Plot params')
name = pb_config.INTEREST_PARAM_NAME
plot_params(name,
result_table,
title=model.full_name,
directory=model.results_path)
logger.info('DONE')
return result_table
def build_model(args, i_cv):
model = get_model(args, GradientBoostingModel)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
# PyTorch import torchvision from torchvision import transforms, datasets, models import torch from torch import optim, cuda from torch.utils.data import DataLoader, sampler import torch.nn as nn from torch_lr_finder import LRFinder from utils.model import get_model, get_dataloaders model = get_model() dataloaders = get_dataloaders() # we will be using negative log likelihood as the loss function criterion = nn.CrossEntropyLoss() # we will be using the SGD optimizer as our optimizer optimizer = optim.SGD(model.fc.parameters(), lr=1e-4) lr_finder = LRFinder(model, optimizer, criterion, device='cuda') lr_finder.range_test(dataloaders['train'], end_lr=1, num_iter=2500) lr_finder.plot() lr_finder.reset()
def train_net(cfg):
data_source = cfg['data_source']
data_params = cfg['data_params']
teacher_params = cfg['teacher_params']
student_params = cfg['student_params']
loss_params = cfg['loss_params']
test_params = cfg['test_params']
opt_params = cfg['opt_params']
# Set data iterators
train_iter, _ = get_rec_data_iterators(data_source['params']['train_db'], '',
cfg['input_shape'], cfg['batch_size'],
data_params, cfg['devices_id'])
devices = [mx.gpu(device_id) for device_id in cfg['devices_id']]
batch_size = cfg['batch_size'] * len(devices)
num_batches = data_source['train_samples_num'] // batch_size
# Set teacher extractor
if teacher_params['type'] == 'insightface-resnet':
teacher_net = get_model(teacher_params, True)
else:
sys.exit('Unsupported teacher net architecture: %s !' % teacher_params['type'])
# Init teacher extractor
teacher_net.load_parameters('%s-%04d.params' % teacher_params['init'], ctx=devices,
allow_missing=False, ignore_extra=False)
logging.info("Teacher extractor parameters were successfully loaded")
teacher_net.hybridize(static_alloc=True, static_shape=True)
# Set student extractor
if student_params['type'] == 'insightface-resnet':
student_net = get_model(student_params, False, 'student_')
else:
sys.exit('Unsupported student net architecture: %s !' % student_params['type'])
# Init student extractor
if student_params['init']:
student_net.load_parameters('%s-%04d.params' % student_params['init'], ctx=devices,
allow_missing=False, ignore_extra=False)
logging.info("Student extractor parameters were successfully loaded")
else:
init_params = [
('.*gamma|.*alpha|.*running_mean|.*running_var', mx.init.Constant(1)),
('.*beta|.*bias', mx.init.Constant(0.0)),
('.*weight', mx.init.Xavier())
]
for mask, initializer in init_params:
student_net.collect_params(mask).initialize(initializer, ctx=devices)
student_net.hybridize(static_alloc=True, static_shape=True)
params = student_net.collect_params()
# Set teacher classifier
teacher_clf = None
if loss_params['HKD']['weight'] > 0.0:
teacher_clf = get_angular_classifier(data_source['num_classes'],
teacher_params['embedding_dim'],
loss_params['classification'])
# init teacher classifier
filename = '%s-%04d.params' % loss_params['HKD']['teacher_init']
teacher_clf.load_parameters(filename, ctx=devices,
allow_missing=False,
ignore_extra=False)
logging.info("Teacher classifier parameters "
"were successfully loaded")
teacher_clf.hybridize(static_alloc=True, static_shape=True)
# Set student classifier
student_clf = None
if loss_params['HKD']['weight'] > 0.0 or loss_params['classification']['weight'] > 0.0:
student_clf = get_angular_classifier(data_source['num_classes'],
student_params['embedding_dim'],
loss_params['classification'],
'student_')
# init student classifier
if loss_params['classification']['student_init']:
filename = '%s-%04d.params' % loss_params['classification']['student_init']
student_clf.load_parameters(filename, ctx=devices,
allow_missing=False,
ignore_extra=False)
logging.info("Student classifier parameters "
"were successfully loaded")
else:
student_clf.initialize(mx.init.Normal(0.01), ctx=devices)
student_clf.hybridize(static_alloc=True, static_shape=True)
params.update(student_clf.collect_params())
# Set losses
L_clf, L_hkd, L_mld = get_losses(loss_params)
# Set train evaluation metrics
eval_metrics_train = init_eval_metrics(loss_params)
# Set optimizer
optimizer = 'sgd'
optimizer_params = {'wd': opt_params['wd'], 'momentum': opt_params['momentum']}
# Set trainer
trainer = gluon.Trainer(params, optimizer, optimizer_params, kvstore='local')
# Initialize test results
test_best_result = {db_name : [0.0, 0] for db_name in test_params['dbs']}
# TRAINING LOOP
iteration = 0
for epoch in range(opt_params['num_epoch']):
tic_epoch = time.time()
# reset metrics
for metric in eval_metrics_train.values():
metric['metric'].reset()
metric['losses'] = []
# update learning rate: step decay
if epoch == 0:
trainer.set_learning_rate(opt_params['lr_base'])
elif epoch > 0 and not epoch % opt_params['lr_epoch_step']:
trainer.set_learning_rate(trainer.learning_rate * opt_params['lr_factor'])
logging.info("Learning rate has been changed to %f" % trainer.learning_rate)
tic_batch = time.time()
for i, batch in enumerate(train_iter):
iteration += 1
# process batch
data, label = unpack_batch(batch)
loss = []
for X, y_gt in zip(data, label):
# get teacher predictions
with autograd.predict_mode():
embeddings_teacher = teacher_net(X)
if teacher_clf:
logits_teacher = teacher_clf(embeddings_teacher, y_gt)
# get student predictions and compute loss
with autograd.record():
embeddings_student = student_net(X)
if student_clf:
logits_student = student_clf(embeddings_student, y_gt)
device_losses = []
# classification loss
if L_clf:
loss_clf = loss_params['classification']['weight'] * \
L_clf(logits_student, y_gt)
device_losses.append(loss_clf)
eval_metrics_train['classification']['losses'].append(loss_clf)
# Hinton's knowledge distillation loss
if L_hkd:
loss_hkd = loss_params['HKD']['weight'] * \
L_hkd(logits_student, logits_teacher)
device_losses.append(loss_hkd)
eval_metrics_train['HKD']['losses'].append(loss_hkd)
# metric learning distillation losses
for name, L, weight in L_mld:
loss_mld = weight * L(embeddings_student, embeddings_teacher)
device_losses.append(loss_mld)
eval_metrics_train[name]['losses'].append(loss_mld)
# aggregate all losses
device_losses = [loss_term.mean() for loss_term in device_losses]
loss.append(mx.nd.add_n(*device_losses))
eval_metrics_train['total']['losses'] = loss
# Backpropagate errors
for l in loss:
l.backward()
trainer.step(batch_size)
# update metrics
for metric in eval_metrics_train.values():
metric['metric'].update(_, metric['losses'])
metric['losses'] = []
# display training statistics
if not (i+1) % cfg['display_period']:
disp_template = 'Epoch[%d/%d] Batch[%d/%d]\tSpeed: %f samples/sec\tlr=%f'
disp_params = [epoch, opt_params['num_epoch'], i+1, num_batches,
batch_size * cfg['display_period'] / (time.time() - tic_batch),
trainer.learning_rate]
for metric in eval_metrics_train.values():
metric_name, metric_score = metric['metric'].get()
disp_template += '\t%s=%f'
disp_params.append(metric_name)
disp_params.append(metric_score)
logging.info(disp_template % tuple(disp_params))
tic_batch = time.time()
if not iteration % cfg['test_period']:
period_idx = iteration // cfg['test_period']
# save model
logging.info("[Epoch %d][Batch %d] "
"Saving network params [%d] at %s" %
(epoch, i, period_idx, cfg['experiment_dir']))
student_net.export('%s/student' % cfg['experiment_dir'], period_idx)
if student_net:
student_clf.export('%s/student-clf' % cfg['experiment_dir'], period_idx)
# test model using outside data
if test_params['dbs']:
logging.info('[Epoch %d] Testing student network ...' % epoch)
# emore bin-files testing
for db_name in test_params['dbs']:
db_path = '%s/%s.bin' % (test_params['dbs_root'], db_name)
data_set = load_bin(db_path, [cfg['input_shape'][1], cfg['input_shape'][2]])
_, _, acc, std, _, _ = test(data_set, student_net, cfg['batch_size'], 10)
if acc > test_best_result[db_name][0]:
test_best_result[db_name] = [acc, period_idx]
logging.info("Epoch[%d] Batch[%d] %s: "
"Accuracy-Flip = %1.5f+-%1.5f "
"(best: %f, at snapshot %04d)" %
(epoch, i+1, db_name, acc, std,
test_best_result[db_name][0],
test_best_result[db_name][1]))
# estimate epoch training speed
throughput = int(batch_size * (i+1) / (time.time() - tic_epoch))
logging.info("[Epoch %d] Speed: %d samples/sec\t"
"Time cost: %f seconds" %
(epoch, throughput, time.time() - tic_epoch))
def main(args, configs):
print("Prepare training ...")
preprocess_config, model_config, train_config = configs
# Get dataset
dataset = Dataset("train.txt",
preprocess_config,
train_config,
model_config,
sort=True,
drop_last=True)
batch_size = train_config["optimizer"]["batch_size"]
group_size = 4 # Set this larger than 1 to enable sorting in Dataset
assert batch_size * group_size < len(dataset)
loader = DataLoader(
dataset,
batch_size=batch_size * group_size,
shuffle=True,
collate_fn=dataset.collate_fn,
)
# Prepare model
model, optimizer = get_model(args, configs, device, train=True)
model = nn.DataParallel(model)
num_param = get_param_num(model)
Loss = FastSpeech2Loss(preprocess_config, model_config).to(device)
print("Number of FastSpeech2 Parameters:", num_param)
# Load checkpoint if exists
if args.restore_path is not None and os.path.isfile(args.restore_path):
checkpoint = torch.load(args.restore_path)
pretrained_dict = checkpoint['model']
if not any(key.startswith('module.') for key in pretrained_dict):
pretrained_dict = {
'module.' + k: v
for k, v in pretrained_dict.items()
}
dem1 = 0
dem2 = 0
model_dict = model.state_dict()
for k, v in pretrained_dict.items():
if k in model_dict and v.size() == model_dict[k].size():
# print('Load weight in ', k)
dem1 += 1
else:
print(f'Module {k} is not same size')
dem2 += 1
dem2 += dem1
print(f'### Load {dem1}/{dem2} modules')
# 1. filter out unnecessary keys
pretrained_dict = {
k: v
for k, v in pretrained_dict.items()
if k in model_dict and v.size() == model_dict[k].size()
}
# 2. overwrite entries in the existing state dict
model_dict.update(pretrained_dict)
# 3. load the new state dict
model.load_state_dict(model_dict)
# model.load_state_dict(checkpoint['model'])
# optimizer.load_state_dict(checkpoint['optimizer'])
print("\n---Model Restored at Step {}---\n".format(args.restore_step))
# Load vocoder
vocoder = get_vocoder(model_config, device)
# Init logger
for p in train_config["path"].values():
os.makedirs(p, exist_ok=True)
train_log_path = os.path.join(train_config["path"]["log_path"], "train")
val_log_path = os.path.join(train_config["path"]["log_path"], "val")
os.makedirs(train_log_path, exist_ok=True)
os.makedirs(val_log_path, exist_ok=True)
train_logger = SummaryWriter(train_log_path)
val_logger = SummaryWriter(val_log_path)
# Training
step = args.restore_step + 1
epoch = 1
grad_acc_step = train_config["optimizer"]["grad_acc_step"]
grad_clip_thresh = train_config["optimizer"]["grad_clip_thresh"]
total_step = train_config["step"]["total_step"]
log_step = train_config["step"]["log_step"]
save_step = train_config["step"]["save_step"]
synth_step = train_config["step"]["synth_step"]
val_step = train_config["step"]["val_step"]
outer_bar = tqdm(total=total_step, desc="Training", position=0)
outer_bar.n = args.restore_step
outer_bar.update()
while True:
inner_bar = tqdm(total=len(loader),
desc="Epoch {}".format(epoch),
position=1)
for batchs in loader:
for batch in batchs:
batch = to_device(batch, device)
# Forward
output = model(*(batch[2:]))
# Cal Loss
losses = Loss(batch, output)
total_loss = losses[0]
# Backward
total_loss = total_loss / grad_acc_step
total_loss.backward()
if step % grad_acc_step == 0:
# Clipping gradients to avoid gradient explosion
nn.utils.clip_grad_norm_(model.parameters(),
grad_clip_thresh)
# Update weights
optimizer.step_and_update_lr()
optimizer.zero_grad()
if step % log_step == 0:
losses = [l.item() for l in losses]
message1 = "Step {}/{}|".format(step, total_step)
message2 = "|Total Loss: {:.4f}|Mel Loss: {:.4f}|Mel PostNet Loss: {:.4f}|Pitch Loss: {:.4f}|Energy Loss: {:.4f}|Duration Loss: {:.4f}|".format(
*losses)
# with open(os.path.join(train_log_path, "log.txt"), "a") as f:
# f.write(message1 + message2 + "\n")
outer_bar.write(message1 + message2)
log(train_logger, step, losses=losses)
if step % synth_step == 0:
output_preidiction = model(*(batch[2:6]))
fig, wav_reconstruction, wav_prediction, tag = synth_one_sample(
batch,
output_preidiction,
vocoder,
model_config,
preprocess_config,
)
log(
train_logger,
fig=fig,
tag="Training/step_{}_{}".format(step, tag),
)
sampling_rate = preprocess_config["preprocessing"][
"audio"]["sampling_rate"]
log(
train_logger,
audio=wav_reconstruction,
sampling_rate=sampling_rate,
tag="Training/step_{}_{}_reconstructed".format(
step, tag),
)
log(
train_logger,
audio=wav_prediction,
sampling_rate=sampling_rate,
tag="Training/step_{}_{}_synthesized".format(
step, tag),
)
if step % val_step == 0:
model.eval()
message = evaluate(model, step, configs, val_logger,
vocoder)
# with open(os.path.join(val_log_path, "log.txt"), "a") as f:
# f.write(message + "\n")
outer_bar.write(message)
model.train()
if step % save_step == 0:
torch.save(
{
"model": model.module.state_dict(),
"optimizer": optimizer._optimizer.state_dict(),
},
os.path.join(
train_config["path"]["ckpt_path"],
"{}.pth.tar".format(step),
),
)
if step == total_step:
quit()
step += 1
outer_bar.update(1)
inner_bar.update(1)
epoch += 1
def build_model(args, i_cv):
model = get_model(args, FeatureModel)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
x_test = 2 * x_test - 1
# convert class vectors to binary class matrices (one-hot encoding)
n_output = 10
y_train = np_utils.to_categorical(y_train, n_output)
y_test = np_utils.to_categorical(y_test, n_output)
print("X_Train: {}\nX_Test: {}\nY_Train: {}\nY_Test: {}" \
.format(x_train.shape, x_test.shape, y_train.shape, y_test.shape))
print("Train Samples: {}, Test Samples: {}".format(x_train.shape[0],
x_test.shape[0]))
arg = parse_args()
# Instantiate Model and load pre-trained weights
model = get_model(arg)
model.load_weights(arg.weights_path)
optimizer = Adam(lr=arg.lr, decay=1e-6)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
training = arg.training
compressing = arg.compressing
# ensure weights are loaded correctly by evaluating the model here and printing the output
# test_loss, test_acc = model.evaluate(x_test, y_test, batch_size=arg.batch_size)
# print("Non-pruned accuracy: {}".format(test_acc))
# convert the layers to masked layers
pruned_model = convert_to_masked_model(model)
def run(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
result_table = []
# LOAD/GENERATE DATA
logger.info('Set up data generator')
pb_config = AP1Config()
seed = config.SEED + i_cv * 5
train_generator = Generator(param_generator, AP1(seed))
valid_generator = AP1(seed + 1)
test_generator = AP1(seed + 2)
# SET MODEL
logger.info('Set up rergessor')
args.net = F3R3(n_in=1, n_out=2)
args.optimizer = get_optimizer(args)
model = get_model(args, Regressor)
model.set_info(BENCHMARK_NAME, i_cv)
flush(logger)
# TRAINING / LOADING
if not args.retrain:
try:
logger.info('loading from {}'.format(model.path))
model.load(model.path)
except Exception as e:
logger.warning(e)
args.retrain = True
if args.retrain:
logger.info('Training {}'.format(model.get_name()))
model.fit(train_generator)
logger.info('Training DONE')
# SAVE MODEL
save_model(model)
# CHECK TRAINING
logger.info('Plot losses')
plot_REG_losses(model)
plot_REG_log_mse(model)
result_row['loss'] = model.losses[-1]
result_row['mse_loss'] = model.mse_losses[-1]
# MEASUREMENT
for mu in pb_config.TRUE_APPLE_RATIO_RANGE:
pb_config.TRUE_APPLE_RATIO = mu
logger.info('Generate testing data')
X_test, y_test, w_test = test_generator.generate(
apple_ratio=pb_config.TRUE_APPLE_RATIO,
n_samples=pb_config.N_TESTING_SAMPLES)
pred, sigma = model.predict(X_test, w_test)
name = pb_config.INTEREST_PARAM_NAME
result_row[name] = pred
result_row[name + _ERROR] = sigma
result_row[name + _TRUTH] = pb_config.TRUE_APPLE_RATIO
logger.info('{} =vs= {} +/- {}'.format(pb_config.TRUE_APPLE_RATIO,
pred, sigma))
result_table.append(result_row.copy())
result_table = pd.DataFrame(result_table)
logger.info('Plot params')
param_names = pb_config.PARAM_NAMES
for name in param_names:
plot_params(name, result_table, model)
logger.info('DONE')
return result_table
def model_summary():
sumry = []
model.get_model().summary(line_length=35, print_fn=lambda x: sumry.append(x))
summary = "\n".join(sumry)
return summary
def main(args, configs):
print("Prepare training ...")
preprocess_config, model_config, train_config = configs
# Get dataset
dataset = Dataset("train.txt",
preprocess_config,
train_config,
sort=True,
drop_last=True)
batch_size = train_config["optimizer"]["batch_size"]
group_size = 4 # Set this larger than 1 to enable sorting in Dataset
assert batch_size * group_size < len(dataset)
loader = DataLoader(
dataset,
batch_size=batch_size * group_size,
shuffle=True,
collate_fn=dataset.collate_fn,
)
with open(
os.path.join(preprocess_config["path"]["preprocessed_path"],
"stats.json")) as f:
stats = json.load(f)
mel_stats = stats["mel"]
# Prepare model
model, optimizer = get_model(args, configs, device, train=True)
model = nn.DataParallel(model)
num_param = get_param_num(model)
Loss = ParallelTacotron2Loss(model_config, train_config).to(device)
print("Number of Parallel Tacotron 2 Parameters:", num_param)
# Load vocoder
vocoder = get_vocoder(model_config, device)
# Init logger
for p in train_config["path"].values():
os.makedirs(p, exist_ok=True)
train_log_path = os.path.join(train_config["path"]["log_path"], "train")
val_log_path = os.path.join(train_config["path"]["log_path"], "val")
os.makedirs(train_log_path, exist_ok=True)
os.makedirs(val_log_path, exist_ok=True)
train_logger = SummaryWriter(train_log_path)
val_logger = SummaryWriter(val_log_path)
# Training
step = args.restore_step + 1
epoch = 1
grad_acc_step = train_config["optimizer"]["grad_acc_step"]
grad_clip_thresh = train_config["optimizer"]["grad_clip_thresh"]
total_step = train_config["step"]["total_step"]
log_step = train_config["step"]["log_step"]
save_step = train_config["step"]["save_step"]
synth_step = train_config["step"]["synth_step"]
val_step = train_config["step"]["val_step"]
outer_bar = tqdm(total=total_step, desc="Training", position=0)
outer_bar.n = args.restore_step
outer_bar.update()
# with torch.autograd.detect_anomaly():
while True:
inner_bar = tqdm(total=len(loader),
desc="Epoch {}".format(epoch),
position=1)
for batchs in loader:
for batch in batchs:
batch = to_device(batch, device, mel_stats)
# Forward
output = model(*(batch[2:]))
# Cal Loss
losses = Loss(batch, output, step)
total_loss = losses[0]
# Backward
total_loss = total_loss / grad_acc_step
total_loss.backward()
if step % grad_acc_step == 0:
# Clipping gradients to avoid gradient explosion
nn.utils.clip_grad_norm_(model.parameters(),
grad_clip_thresh)
# Update weights
optimizer.step_and_update_lr()
optimizer.zero_grad()
if step % log_step == 0:
losses = [l.item() for l in losses]
message1 = "Step {}/{}, ".format(step, total_step)
message2 = "Total Loss: {:.4f}, Mel Loss: {:.4f}, Duration Loss: {:.4f}, KL Loss: {:.4f}".format(
*losses)
with open(os.path.join(train_log_path, "log.txt"),
"a") as f:
f.write(message1 + message2 + "\n")
outer_bar.write(message1 + message2)
log(train_logger, step, losses=losses)
if step % synth_step == 0:
fig, wav_reconstruction, wav_prediction, tag = synth_one_sample(
batch,
output,
vocoder,
model_config,
preprocess_config,
mel_stats,
)
log(
train_logger,
fig=fig,
tag="Training/step_{}_{}".format(step, tag),
)
sampling_rate = preprocess_config["preprocessing"][
"audio"]["sampling_rate"]
log(
train_logger,
audio=wav_reconstruction,
sampling_rate=sampling_rate,
tag="Training/step_{}_{}_reconstructed".format(
step, tag),
)
log(
train_logger,
audio=wav_prediction,
sampling_rate=sampling_rate,
tag="Training/step_{}_{}_synthesized".format(
step, tag),
)
if step % val_step == 0:
model.eval()
message = evaluate(model, step, configs, val_logger,
vocoder, len(losses), mel_stats)
with open(os.path.join(val_log_path, "log.txt"), "a") as f:
f.write(message + "\n")
outer_bar.write(message)
model.train()
if step % save_step == 0:
torch.save(
{
"model": model.module.state_dict(),
"optimizer": optimizer._optimizer.state_dict(),
},
os.path.join(
train_config["path"]["ckpt_path"],
"{}.pth.tar".format(step),
),
)
if step == total_step:
quit()
step += 1
outer_bar.update(1)
inner_bar.update(1)
epoch += 1
if train_on_gpu:
gpu_count = cuda.device_count()
print(f'{gpu_count} gpus detected.')
if gpu_count > 1:
multi_gpu = True
else:
multi_gpu = False
# set random seeds
torch.manual_seed(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(42)
# get model, loss function, optimizer, and scheduler
model = get_model(train_on_gpu, multi_gpu)
criterion, optimizer, scheduler = get_optimizer_scheduler_and_criterion(model)
dataloaders = get_dataloaders()
# start training for first cycle of training
model, history = train_model(model,
criterion,
optimizer,
scheduler,
dataloaders['train'],
dataloaders['val'],
save_file_name=save_file_name,
checkpoint_file_name=checkpoint_file_name,
early_stopping_patience=50,
def build_model(args, i_cv):
args.net = ARCHI(n_in=31, n_out=2, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
model = get_model(args, Model)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
eval_files = glob.glob(
os.path.join(args.data_folder, 'eval', 'UNET_256_[A-Z]*.gz'))
print(len(train_files))
training = processing.get_training_dataset(train_files,
ftDict=FEATURES_DICT,
buff=args.buffer,
batch=args.batch)
evaluation = processing.get_eval_dataset(eval_files, ftDict=FEATURES_DICT)
# get the run context
run = Run.get_context()
# build the model
m = model.get_model(depth=len(BANDS), optim=OPTIMIZER, loss=LOSS, mets=METRICS)
# compile the model with our loss fxn, metrics, and optimizer
m.compile(optimizer=OPTIMIZER, loss=LOSS, metrics=METRICS)
# create special folders './outputs' and './logs' which automatically get saved
os.makedirs('outputs', exist_ok=True)
os.makedirs('logs', exist_ok=True)
out_dir = './outputs'
log_dir = './logs'
# get the current time
now = datetime.now()
date = now.strftime("%d%b%y")
date
def build_model(args, i_cv):
args.net = ARCHI(n_in=1, n_out=2, n_unit=args.n_unit)
args.optimizer = get_optimizer(args)
model = get_model(args, TangentPropClassifier)
model.set_info(DATA_NAME, BENCHMARK_NAME, i_cv)
return model
def run(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
result_table = []
# LOAD/GENERATE DATA
logger.info('Set up data generator')
pb_config = AP1Config()
seed = config.SEED + i_cv * 5
train_generator = AP1(seed)
valid_generator = AP1(seed+1)
test_generator = AP1(seed+2)
# SET MODEL
logger.info('Set up classifier')
model = get_model(args, GradientBoostingModel)
model.set_info(BENCHMARK_NAME, i_cv)
flush(logger)
# TRAINING / LOADING
if not args.retrain:
try:
logger.info('loading from {}'.format(model.path))
model.load(model.path)
except Exception as e:
logger.warning(e)
args.retrain = True
if args.retrain:
logger.info('Generate training data')
X_train, y_train, w_train = train_generator.generate(
apple_ratio=pb_config.CALIBRATED_APPLE_RATIO,
n_samples=pb_config.N_TRAINING_SAMPLES)
logger.info('Training {}'.format(model.get_name()))
model.fit(X_train, y_train, w_train)
logger.info('Training DONE')
# SAVE MODEL
save_model(model)
# CHECK TRAINING
logger.info('Generate validation data')
X_valid, y_valid, w_valid = valid_generator.generate(
apple_ratio=pb_config.CALIBRATED_APPLE_RATIO,
n_samples=pb_config.N_VALIDATION_SAMPLES)
logger.info('Plot distribution of the score')
plot_valid_distrib(model, X_valid, y_valid, classes=("pears", "apples"))
result_row['valid_accuracy'] = model.score(X_valid, y_valid)
# MEASUREMENT
n_bins = 10
compute_summaries = ClassifierSummaryComputer(model, n_bins=n_bins)
for mu in pb_config.TRUE_APPLE_RATIO_RANGE:
pb_config.TRUE_APPLE_RATIO = mu
logger.info('Generate testing data')
X_test, y_test, w_test = test_generator.generate(
apple_ratio=pb_config.TRUE_APPLE_RATIO,
n_samples=pb_config.N_TESTING_SAMPLES)
logger.info('Set up NLL computer')
compute_nll = AP1NLL(compute_summaries, valid_generator, X_test, w_test)
logger.info('Plot summaries')
extension = '-mu={:1.1f}'.format(pb_config.TRUE_APPLE_RATIO)
plot_summaries( model, n_bins, extension,
X_valid, y_valid, w_valid,
X_test, w_test, classes=('pears', 'apples', 'fruits') )
# NLL PLOTS
logger.info('Plot NLL around minimum')
plot_apple_ratio_around_min(compute_nll,
pb_config.TRUE_APPLE_RATIO,
model,
extension)
# MINIMIZE NLL
logger.info('Prepare minuit minimizer')
minimizer = get_minimizer(compute_nll)
fmin, params = estimate(minimizer)
params_truth = [pb_config.TRUE_APPLE_RATIO]
print_params(params, params_truth)
register_params(params, params_truth, result_row)
result_row['is_mingrad_valid'] = minimizer.migrad_ok()
result_row.update(fmin)
result_table.append(result_row.copy())
result_table = pd.DataFrame(result_table)
logger.info('Plot params')
param_names = pb_config.PARAM_NAMES
for name in param_names:
plot_params(name, result_table, title=model.full_name, directory=model.path)
logger.info('DONE')
return result_table
def get(self, name):
return get_model(name)
