def test_neural_net( trained_network_filename, testing_data_filename, results_filename):
num_input_nodes, num_hidden_nodes, num_output_nodes, weights = utils.load_neural_net(trained_network_filename)
num_features, num_outputs, test_dataset = utils.load_data(testing_data_filename)
network = create_network(num_hidden_nodes, num_output_nodes, weights)
# confusion_matrices[output_idx][ predicted ][ expected ]
confusion_matrices = [
[
[0 , 0],
[0 , 0]
]
for _ in range(num_outputs)
]
# num_features, num_outputs, dataset = load_data(data_filename)
for row in test_dataset:
prediction = predict(network, row)[-num_outputs:]
expected = row[-num_outputs:]
print("=="*10)
print( prediction )
print( expected )
for i in range(num_outputs):
confusion_matrices[i][ int(round( prediction[i] )) ][ int(expected[i]) ] += 1.0
outputs_metrics = utils.calc_metrics( confusion_matrices )
utils.save_metrics( results_filename, outputs_metrics )
return
def pred():
batch_size = 32
output_dir = ps.output
output_dir = output_dir[:-1] if output_dir.endswith(os.sep) else output_dir
if not os.path.exists(output_dir): os.mkdir(output_dir)
rm_top = ps.rm_top
data = load_data_from_tfr()
num_samples = data['num_samples']
num_batches = num_samples // batch_size
batch_left = num_samples % batch_size
if batch_left != 0: num_batches += 1
models = get_model_list()
num_models = len(models)
res = model(is_train=False, show_layers_info=False)
predictions = np.zeros((num_models, num_samples)) #(5, 80)
for idx in range(num_models):
sess = None
with tf.Session() as sess:
load_and_assign_npz(sess, models[idx], res['net'])
for batchIdx in range(num_batches):
if batchIdx == num_batches - 1:
batch_length = batch_left if batch_left else batch_size
batch_X = data['X'][batchIdx * batch_size:]
else:
batch_length = batch_size
batch_X = data['X'][batchIdx * batch_size:(batchIdx + 1) *
batch_size]
prediction = sess.run([res['prediction']],
feed_dict={res['x']: batch_X})
predictions[idx, batchIdx * batch_size:batchIdx * batch_size +
batch_length] = np.squeeze(prediction)
# predictions = np.mean(predictions, axis=0)
predictions = cond_mean(predictions, rm_top=rm_top)
predictions = predictions * 100
TFRecordFileBuilder._write_dict(
"{}{}metrics-results.csv".format(output_dir, os.sep),
calc_metrics(data['y'], predictions))
write2file("{}{}predictions-results.csv".format(output_dir, os.sep),
data['y'], predictions)
def __metrics_init(self, fixed, moving):
step = 0
self.writer.set_step(step)
residuals = self.losses['data']['loss'].map(fixed['im'], moving['im'])
residuals_masked = residuals[fixed['mask']]
log_hist_res(self.writer,
residuals_masked,
self.losses['data']['loss'],
model='VI')
log_images(self.writer, fixed['im'], moving['im'], moving['im'])
# metrics
ASD, DSC = calc_metrics(fixed['seg'], moving['seg'],
self.structures_dict, self.im_spacing)
for structure_idx, structure in enumerate(self.structures_dict):
ASD_val, DSC_val = ASD[0][structure_idx], DSC[0][structure_idx]
self.metrics.update(f'VI/train/ASD/{structure}', ASD_val)
self.metrics.update(f'VI/train/DSC/{structure}', DSC_val)
def test(test_loader, model, criterion, step, device):
model.eval()
test_len = len(test_loader)
rand = random.randint(1, test_len)
outputs, targets = [], []
START_FLAG = True
with torch.no_grad():
for batch_idx, (data, target) in enumerate(test_loader):
data, target = data.to(device), target.to(device)
output, reconstructions = model(data)
if START_FLAG:
outputs = output
targets = target
START_FLAG = False
else:
outputs = torch.cat([outputs, output])
targets = torch.cat([targets, target])
if batch_idx == rand and args.add_decoder:
log_reconstruction_sample(test_writer, data, reconstructions,
step)
act_loss, rec_loss, tot_loss = criterion(outputs, targets, None, None)
test_loss = act_loss.item()
acc, rec, f1, hamm, emr = calc_metrics(outputs, targets)
# Logging
test_writer.add_scalar('Loss/BCE', test_loss, step)
test_writer.add_scalar('Metrics/Precision', acc, step)
test_writer.add_scalar('Metrics/Recall', rec, step)
test_writer.add_scalar('Metrics/F1-score', f1, step)
test_writer.add_scalar('Metrics/HammingScore', hamm, step)
test_writer.add_scalar('Metrics/ExactMatchRatio', emr, step)
log_heatmap(test_writer, outputs, targets, step)
print(
'\nTest set: Average loss: {:.6f}, F1-Score: {:.6f}, ExactMatch: {:.6f} \n'
.format(test_loss, f1, emr))
return test_loss, f1
def single_target_training(cfg, X_train, y_train, X_test, logger):
model_name = list(cfg["model"].keys())[0]
metric = utils.get_metric(cfg)
metric_name = cfg["metric"]["name"]
experiment_id = utils.get_experiment_id(cfg)
run_name = cfg["mlflow"]["run_name"]
logger.info(f"experiment config: {run_name}")
logger.info(
f"CV method: {cfg['split']['name']} {cfg['split']['params']['n_splits']}-Fold"
)
with mlflow.start_run(run_name=run_name, experiment_id=experiment_id):
trainer = get_trainer(cfg, model_name, X_train, y_train, X_test)
y_oof, models, y_pred = training_step(trainer)
metrics = utils.calc_metrics(cfg, metric, y_train.values, y_oof)
fig = utils.plot_feature_importance(models, X_train, model_name)
logger.info(f"CV score : {metrics[metric_name]}")
utils.mlflow_logger(cfg, metrics, fig, targets=None)
return y_pred
def multi_target_training(cfg, X_train, y_trains, X_test, logger):
model_name = list(cfg["model"].keys())[0]
targets = cfg["training"]["targets"]
y_oof = np.zeros((len(X_train), len(targets)))
y_pred = np.zeros((len(X_test), len(targets)))
metric = utils.get_metric(cfg)
metric_name = cfg["metric"]["name"]
figs = []
experiment_id = utils.get_experiment_id(cfg)
run_name = cfg["mlflow"]["run_name"]
logger.info(f"experiment config: {run_name}")
logger.info(
f"CV method: {cfg['split']['name']} {cfg['split']['params']['n_splits']}-Fold"
)
with mlflow.start_run(run_name=run_name, experiment_id=experiment_id):
for i, target in enumerate(targets):
logger.info(f"Training for {target}")
y_train = y_trains[target]
trainer = get_trainer(cfg, model_name, X_train, y_train, X_test)
y_oof_, models, y_pred_ = training_step(trainer)
y_oof[:, i] = y_oof_
y_pred[:, i] = y_pred_
fig = utils.plot_feature_importance(models, X_train, model_name,
target)
figs.append(fig)
metrics = utils.calc_metrics(cfg, metric, y_trains.values, y_oof)
logger.info(f"CV score : {metrics[metric_name]}")
utils.mlflow_logger(cfg, metrics, figs, targets)
return y_pred
def _test_VI(self, fixed, moving, var_params_q_v):
"""
metrics
"""
samples = torch.zeros([self.no_samples_VI_test, 3,
*self.dims]) # samples used in the evaluation
for test_sample_no in range(1, self.no_samples_VI_test + 1):
self.writer.set_step(test_sample_no)
v_sample = sample_q_v(var_params_q_v, no_samples=1)
v_sample_smoothed = SobolevGrad.apply(v_sample, self.S,
self.padding)
transformation, displacement = self.transformation_module(
v_sample_smoothed)
samples[test_sample_no - 1] = displacement.clone().cpu()
no_non_diffeomorphic_voxels, log_det_J = calc_no_non_diffeomorphic_voxels(
transformation, self.diff_op)
self.metrics.update('VI/test/no_non_diffeomorphic_voxels',
no_non_diffeomorphic_voxels)
im_moving_warped = self.registration_module(
moving['im'], transformation)
seg_moving_warped = self.registration_module(
moving['seg'], transformation)
ASD, DSC = calc_metrics(fixed['seg'], seg_moving_warped,
self.structures_dict, self.im_spacing)
for structure_idx, structure in enumerate(self.structures_dict):
ASD_val, DSC_val = ASD[0][structure_idx], DSC[0][structure_idx]
self.metrics.update(f'VI/test/ASD/{structure}', ASD_val)
self.metrics.update(f'VI/test/DSC/{structure}', DSC_val)
save_sample(self.save_dirs, self.im_spacing, test_sample_no,
im_moving_warped, displacement, log_det_J, 'VI')
self.logger.info(
'\nsaving the displacement and warped moving image corresponding to the mean of the approximate variational posterior..'
)
mu_v = var_params_q_v['mu']
mu_v_smoothed = SobolevGrad.apply(mu_v, self.S, self.padding)
transformation, displacement = self.transformation_module(
mu_v_smoothed)
im_moving_warped = self.registration_module(moving['im'],
transformation)
save_variational_posterior_mean(self.save_dirs, self.im_spacing,
im_moving_warped, displacement)
self.logger.info(
'\ncalculating sample standard deviation of the displacement..')
mean, std_dev = calc_posterior_statistics(samples)
log_displacement_mean_and_std_dev(self.writer, mean, std_dev, 'VI')
save_displacement_mean_and_std_dev(self.logger, self.save_dirs,
self.im_spacing, mean, std_dev,
moving['mask'], 'VI')
"""
speed
"""
no_samples_VI_speed_test = 100
start = datetime.now()
for VI_test_sample_no in range(1, no_samples_VI_speed_test + 1):
v_sample = sample_q_v(var_params_q_v, no_samples=1)
v_sample_smoothed = SobolevGrad.apply(v_sample, self.S,
self.padding)
transformation, displacement = self.transformation_module(
v_sample_smoothed)
im_moving_warped = self.registration_module(
moving['im'], transformation)
seg_moving_warped = self.registration_module(
moving['seg'], transformation)
stop = datetime.now()
VI_sampling_speed = no_samples_VI_speed_test / (stop -
start).total_seconds()
self.logger.info(
f'\nVI sampling speed: {VI_sampling_speed:.2f} samples/sec')
def main():
# setting arguments
parser = argparse.ArgumentParser(description='Test arguments')
parser.add_argument('--opt', type=str, required=True, help='path to test yaml file')
parser.add_argument('--name', type=str, required=True, help='test log file name')
parser.add_argument('--dataset_name', type=str, default=None)
parser.add_argument('--scale', type=int, required=True)
parser.add_argument('--gpu_ids', type=str, default=None, help='which gpu to use')
parser.add_argument('--which_model', type=str, required=True, help='which pretrained model')
parser.add_argument('--pretrained', type=str, required=True, help='pretrained path')
args = parser.parse_args()
args, lg = test_parse(args)
pn = 50
half_pn = pn//2
lg.info('\n' + '-'*pn + 'General INFO' + '-'*pn)
# create test dataloader
test_loader_list = []
for i in range(len(args['dataset_list'])):
# get single dataset and dataloader
single_dataset_args = copy.deepcopy(args)
single_dataset_args['datasets']['test']['dataroot_HR'] = single_dataset_args['datasets']['test']['dataroot_HR'][i]
single_dataset_args['datasets']['test']['dataroot_LR'] = single_dataset_args['datasets']['test']['dataroot_LR'][i]
test_dataset = create_dataset(single_dataset_args['datasets']['test'])
test_loader = create_loader(test_dataset, args['datasets']['test'])
test_loader_list.append(test_loader)
# create model
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = create_model(args['networks']).to(device)
lg.info('Create model: [{}]'.format(args['networks']['which_model']))
scale = args['scale']
# calc number of parameters
in_ = torch.randn(1, 3, round(720/scale), round(1280/scale)).to(device)
params, GFlops = summary(model, in_)
lg.info('Total parameters: [{:.3f}M], GFlops: [{:.4f}G]'.format(params / 1e6, GFlops / 1e9))
# load pretrained model
state_dict = torch.load(args['networks']['pretrained'])
new_state_dict = {}
for k, v in state_dict.items():
if k[:7] == 'module.':
new_state_dict[k[7:]] = v
else:
new_state_dict[k] = v
model.load_state_dict(new_state_dict, strict=True)
lg.info('Load pretrained from: [{}]'.format(args['networks']['pretrained']))
for i, test_loader in enumerate(test_loader_list):
dataset_name = args['dataset_list'][i]
l = (12-len(dataset_name)) // 2
e = len(dataset_name) % 2
lg.info('\n\n' + '-'*(pn+l) + dataset_name + '-'*(pn+l+e))
lg.info('Number of [{}] images: [{}]'.format(dataset_name, len(test_loader)))
# calculate cuda time
avg_test_time = 0.0
if args['calc_cuda_time'] and 'Set5' in dataset_name:
lg.info('Start calculating cuda time...')
avg_test_time = calc_cuda_time(test_loader, model)
lg.info('Average cuda time on [{}]: [{:.5f}ms]'.format(dataset_name, avg_test_time))
# calucate psnr and ssim
psnr_list = []
ssim_list = []
model.eval()
for iter, data in enumerate(test_loader):
lr = data['LR'].to(device)
hr = data['HR']
# calculate evaluation metrics
with torch.no_grad():
sr = model(lr)
#save(args['networks']['which_model'], dataset_name, data['filename'][0], tensor2np(sr))
psnr, ssim = calc_metrics(tensor2np(sr), tensor2np(hr), crop_border=scale, test_Y=True)
psnr_list.append(psnr)
ssim_list.append(ssim)
lg.info('[{:03d}/{:03d}] || PSNR/SSIM: {:.2f}/{:.4f} || {}'.format(iter+1, len(test_loader), psnr, ssim, data['filename']))
avg_psnr = sum(psnr_list) / len(psnr_list)
avg_ssim = sum(ssim_list) / len(ssim_list)
if avg_test_time > 0:
lg.info('Average PSNR: {:.2f} Average SSIM: {:.4f}, Average time: {:.5f}ms'.format(avg_psnr, avg_ssim, avg_test_time))
else:
lg.info('Average PSNR: {:.2f} Average SSIM: {:.4f}'.format(avg_psnr, avg_ssim))
lg.info('\n' + '-'*pn + '---Finish---' + '-'*pn)
def train(model,
writer,
seed,
k=100,
alpha=0.6,
lr=0.002,
num_epochs=150,
batch_size=64,
n_classes=10,
max_epochs=80,
max_val=1.):
# prepare data
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(cfg['m'], cfg['s'])])
train_dataset, val_dataset = prepare_mnist(root='~/datasets/MNIST',
transform=transform)
ntrain = len(train_dataset)
# build model and feed to GPU
model.cuda()
# make data loaders
train_loader, val_loader, indices = utils.sample_train(train_dataset,
val_dataset,
batch_size,
k,
n_classes,
seed,
shuffle_train=False)
# setup param optimization
optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.99))
# model.train()
Z = torch.zeros(ntrain, n_classes).float().cuda() # intermediate values
z = torch.zeros(ntrain, n_classes).float().cuda() # temporal outputs
outputs = torch.zeros(ntrain, n_classes).float().cuda() # current outputs
losses = []
suplosses = []
unsuplosses = []
best_loss = 30.0
for epoch in range(num_epochs):
t = timer()
print('\nEpoch: {}'.format(epoch + 1))
model.train()
# evaluate unsupervised cost weight
w = utils.weight_scheduler(epoch, max_epochs, max_val, 5, k, 60000)
w = torch.tensor(w, requires_grad=False).cuda()
print('---------------------')
# targets change only once per epoch
for i, (images, labels) in enumerate(train_loader):
batch_size = images.size(
0) # retrieve batch size again cause drop last is false
images = images.cuda()
labels = labels.requires_grad_(False).cuda()
optimizer.zero_grad()
out = model(images)
zcomp = z[i * batch_size:(i + 1) * batch_size]
# zcomp = torch.tensor(zcomp, requires_grad=False)
zcomp.requires_grad_(False)
loss, suploss, unsuploss, nbsup = utils.temporal_losses(
out, zcomp, w, labels)
# save outputs
outputs[i * batch_size:(i + 1) * batch_size] = out.clone().detach()
losses.append(loss.item())
suplosses.append(nbsup * suploss.item())
unsuplosses.append(unsuploss.item())
# backprop
loss.backward()
optimizer.step()
# print loss every 100 steps
# if (i + 1) % 100 == 0:
# print('Step [%d/%d], Loss: %.6f, Time: %.2f s' % (i+1, len(train_dataset) // batch_size,
# float(np.mean(losses)), timer()-t))
loss_mean = np.mean(losses)
supl_mean = np.mean(suplosses)
unsupl_mean = np.mean(unsuplosses)
writer.add_scalar('total loss', loss_mean, (epoch + 1) * ntrain)
print(
'Epoch [%d/%d], Loss: %.6f, Supervised Loss: %.6f, Unsupervised Loss: %.6f, Time: %.2f'
% (epoch + 1, num_epochs, float(loss_mean), float(supl_mean),
float(unsupl_mean), timer() - t))
writer.add_scalar('supervised loss', supl_mean, (epoch + 1) * ntrain)
writer.add_scalar('unsupervised loss', unsupl_mean,
(epoch + 1) * ntrain)
Z = alpha * Z + (1. - alpha) * outputs
z = Z * (1. / (1. - alpha**(epoch + 1)))
if loss_mean < best_loss:
best_loss = loss_mean
torch.save({'state_dict': model.state_dict()}, 'model_best.pth')
model.eval()
acc = utils.calc_metrics(model, val_loader)
writer.add_scalar('Acc', acc, (epoch + 1) * ntrain)
print('Acc : %.2f' % acc)
# test best model
checkpoint = torch.load('model_best.pth')
model.load_state_dict(checkpoint['state_dict'])
model.eval()
acc_best = utils.calc_metrics(model, val_loader)
print('Acc (best model): %.2f' % acc_best)
def train(model,
seed,
k=100,
alpha=0.6,
lr=0.002,
beta2=0.99,
num_epochs=150,
batch_size=100,
drop=0.5,
std=0.15,
fm1=16,
fm2=32,
divide_by_bs=False,
w_norm=False,
data_norm='pixelwise',
early_stop=None,
c=300,
n_classes=10,
max_epochs=80,
max_val=30.,
ramp_up_mult=-5.,
n_samples=60000,
print_res=True,
**kwargs):
#pdb.set_trace()
writer = SummaryWriter(comment='Semi-supervised')
# retrieve data
train_dataset, test_dataset = prepare_mnist()
ntrain = len(train_dataset)
# build model
model.cuda()
# make data loaders
train_loader, test_loader, indices = sample_train(train_dataset,
test_dataset,
batch_size,
k,
n_classes,
seed,
shuffle_train=False)
# setup param optimization
optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.99))
# train
model.train()
losses = []
sup_losses = []
unsup_losses = []
best_loss = 20.
Z = torch.zeros(ntrain, n_classes).float().cuda() # intermediate values
z = torch.zeros(ntrain, n_classes).float().cuda() # temporal outputs
outputs = torch.zeros(ntrain, n_classes).float().cuda() # current outputs
for epoch in range(num_epochs):
t = timer()
# evaluate unsupervised cost weight
w = weight_schedule(epoch, max_epochs, max_val, ramp_up_mult, k,
n_samples)
if (epoch + 1) % 10 == 0:
print 'unsupervised loss weight : {}'.format(w)
# turn it into a usable pytorch object
w = torch.autograd.Variable(torch.FloatTensor([w]).cuda(),
requires_grad=False)
l = []
supl = []
unsupl = []
#pdb.set_trace()
for i, (images, labels) in enumerate(train_loader):
images = Variable(images.cuda())
labels = Variable(labels.cuda(), requires_grad=False)
# get output and calculate loss
optimizer.zero_grad()
out = model(images)
zcomp = Variable(z[i * batch_size:(i + 1) * batch_size],
requires_grad=False)
loss, suploss, unsuploss, nbsup = temporal_loss(
out, zcomp, w, labels)
# save outputs and losses
outputs[i * batch_size:(i + 1) * batch_size] = out.data.clone()
l.append(loss.data[0])
supl.append(nbsup * suploss.data[0])
unsupl.append(unsuploss.data[0])
# backprop
loss.backward()
optimizer.step()
# print loss
if (epoch + 1) % 1 == 0:
if i + 1 == 2 * c:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.6f, Time (this epoch): %.2f s'
% (epoch + 1, num_epochs, i + 1, len(train_dataset) //
batch_size, np.mean(l), timer() - t))
elif (i + 1) % c == 0:
print('Epoch [%d/%d], Step [%d/%d], Loss: %.6f' %
(epoch + 1, num_epochs, i + 1,
len(train_dataset) // batch_size, np.mean(l)))
# update temporal ensemble
Z = alpha * Z + (1. - alpha) * outputs
z = Z * (1. / (1. - alpha**(epoch + 1)))
# handle metrics, losses, etc.
eloss = np.mean(l)
losses.append(eloss)
sup_losses.append(
(1. / k) *
np.sum(supl)) # division by 1/k to obtain the mean supervised loss
unsup_losses.append(np.mean(unsupl))
writer.add_scalar('Train_loss', losses[epoch], epoch)
writer.add_scalar('Supervised_loss', sup_losses[epoch], epoch)
writer.add_scalar('Unsupervised_loss', unsup_losses[epoch], epoch)
# saving model
if eloss < best_loss:
best_loss = eloss
torch.save({'state_dict': model.state_dict()},
'model_best.pth.tar')
writer.add_graph(model, (images, ))
writer.close()
# test
model.eval()
acc = calc_metrics(model, test_loader)
if print_res:
print 'Accuracy of the network on the 10000 test images: %.2f %%' % (
acc)
# test best model
checkpoint = torch.load('model_best.pth.tar')
model.load_state_dict(checkpoint['state_dict'])
model.eval()
acc_best = calc_metrics(model, test_loader)
if print_res:
print 'Accuracy of the network (best model) on the 10000 test images: %.2f %%' % (
acc_best)
return acc, acc_best, losses, sup_losses, unsup_losses, indices
def main():
args = get_arguments()
with open(args.model_params, 'r') as f:
model_params = json.load(f)
with open(args.training_params, 'r') as f:
train_params = json.load(f)
try:
directories = validate_directories(args)
except ValueError as e:
print('Some arguments are wrong:')
print(str(e))
return
logdir = directories['logdir']
restore_from = directories['restore_from']
# Even if we restored the model, we will treat it as new training
# if the trained model is written into an arbitrary location.
is_overwritten_training = logdir != restore_from
receptive_field = WaveNetModel.calculate_receptive_field(
model_params['filter_width'], model_params['dilations'],
model_params['initial_filter_width'])
# Save arguments and model params into file
save_run_config(args, receptive_field, STARTED_DATESTRING, logdir)
# Create coordinator.
coord = tf.train.Coordinator()
# Create data loader.
with tf.name_scope('create_inputs'):
reader = WavMidReader(data_dir=args.data_dir_train,
coord=coord,
audio_sample_rate=model_params['audio_sr'],
receptive_field=receptive_field,
velocity=args.velocity,
sample_size=args.sample_size,
queues_size=(10, 10 * args.batch_size))
data_batch = reader.dequeue(args.batch_size)
# Create model.
net = WaveNetModel(
batch_size=args.batch_size,
dilations=model_params['dilations'],
filter_width=model_params['filter_width'],
residual_channels=model_params['residual_channels'],
dilation_channels=model_params['dilation_channels'],
skip_channels=model_params['skip_channels'],
output_channels=model_params['output_channels'],
use_biases=model_params['use_biases'],
initial_filter_width=model_params['initial_filter_width'])
input_data = tf.placeholder(dtype=tf.float32,
shape=(args.batch_size, None, 1))
input_labels = tf.placeholder(dtype=tf.float32,
shape=(args.batch_size, None,
model_params['output_channels']))
loss, probs = net.loss(input_data=input_data,
input_labels=input_labels,
pos_weight=train_params['pos_weight'],
l2_reg_str=train_params['l2_reg_str'])
optimizer = optimizer_factory[args.optimizer](
learning_rate=train_params['learning_rate'],
momentum=train_params['momentum'])
trainable = tf.trainable_variables()
optim = optimizer.minimize(loss, var_list=trainable)
# Set up logging for TensorBoard.
writer = tf.summary.FileWriter(logdir)
writer.add_graph(tf.get_default_graph())
run_metadata = tf.RunMetadata()
summaries = tf.summary.merge_all()
histograms = tf.summary.merge_all(key=HKEY)
# Separate summary ops for validation, since they are
# calculated only once per evaluation cycle.
with tf.name_scope('validation_summaries'):
metric_summaries = metrics_empty_dict()
metric_value = tf.placeholder(tf.float32)
for name in metric_summaries.keys():
metric_summaries[name] = tf.summary.scalar(name, metric_value)
images_buffer = tf.placeholder(tf.string)
images_batch = tf.stack([
tf.image.decode_png(images_buffer[0], channels=4),
tf.image.decode_png(images_buffer[1], channels=4),
tf.image.decode_png(images_buffer[2], channels=4)
])
images_summary = tf.summary.image('estim', images_batch)
audio_data = tf.placeholder(tf.float32)
audio_summary = tf.summary.audio('input', audio_data,
model_params['audio_sr'])
# Set up session
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.trainable_variables(),
max_to_keep=args.max_checkpoints)
# Trainer for keeping best validation-performing model
# and optional early stopping.
trainer = Trainer(sess, logdir, train_params['early_stop_limit'], 0.999)
try:
saved_global_step = load(saver, sess, restore_from)
if is_overwritten_training or saved_global_step is None:
# The first training step will be saved_global_step + 1,
# therefore we put -1 here for new or overwritten trainings.
saved_global_step = -1
except:
print('Something went wrong while restoring checkpoint. '
'Training will be terminated to avoid accidentally '
'overwriting the previous model.')
raise
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
reader.start_threads(sess)
step = None
last_saved_step = saved_global_step
try:
for step in range(saved_global_step + 1, train_params['num_steps']):
waveform, pianoroll = sess.run([data_batch[0], data_batch[1]])
feed_dict = {input_data: waveform, input_labels: pianoroll}
# Reload switches from file on each step
with open(RUNTIME_SWITCHES, 'r') as f:
switch = json.load(f)
start_time = time.time()
if switch['store_meta'] and step % switch['store_every'] == 0:
# Slow run that stores extra information for debugging.
print('Storing metadata')
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
summary, loss_value, _ = sess.run([summaries, loss, optim],
feed_dict=feed_dict,
options=run_options,
run_metadata=run_metadata)
writer.add_summary(summary, step)
writer.add_run_metadata(run_metadata,
'step_{:04d}'.format(step))
tl = timeline.Timeline(run_metadata.step_stats)
timeline_path = os.path.join(logdir, 'timeline.trace')
with open(timeline_path, 'w') as f:
f.write(tl.generate_chrome_trace_format(show_memory=True))
else:
summary, loss_value, _ = sess.run([summaries, loss, optim],
feed_dict=feed_dict)
writer.add_summary(summary, step)
duration = time.time() - start_time
print('step {:d} - loss = {:.3f}, ({:.3f} sec/step)'.format(
step, loss_value, duration))
if step % switch['checkpoint_every'] == 0:
save(saver, sess, logdir, step)
last_saved_step = step
# Evaluate model performance on validation data
if step % switch['evaluate_every'] == 0:
if switch['histograms']:
hist_summary = sess.run(histograms)
writer.add_summary(hist_summary, step)
print('evaluating...')
stats = 0, 0, 0, 0, 0, 0
est = np.empty([0, model_params['output_channels']])
ref = np.empty([0, model_params['output_channels']])
b_data, b_labels, b_cntr = (np.empty(
(0, args.sample_size + receptive_field - 1,
1)), np.empty((0, model_params['output_channels'])),
args.batch_size)
# if (batch_size * sample_size > valid_data) single_pass() again
while est.size == 0: # and ref.size == 0 and sum(stats) == 0 ...
for data, labels in reader.single_pass(
sess, args.data_dir_valid):
# cumulate batch
if b_cntr > 1:
b_data, b_labels, decr = cumulateBatch(
data, labels, b_data, b_labels)
b_cntr -= decr
continue
elif args.batch_size > 1:
b_data, b_labels, decr = cumulateBatch(
data, labels, b_data, b_labels)
if not decr:
continue
data = b_data
labels = b_labels
# reset batch cumulation variables
b_data, b_labels, b_cntr = (
np.empty(
(0, args.sample_size + receptive_field - 1,
1)),
np.empty((0, model_params['output_channels'])),
args.batch_size)
predictions = sess.run(probs,
feed_dict={input_data: data})
# Aggregate sums for metrics calculation
stats_chunk = calc_stats(predictions, labels,
args.threshold)
stats = tuple(
[sum(x) for x in zip(stats, stats_chunk)])
est = np.append(est, predictions, axis=0)
ref = np.append(ref, labels, axis=0)
metrics = calc_metrics(None, None, None, stats=stats)
write_metrics(metrics, metric_summaries, metric_value, writer,
step, sess)
trainer.check(metrics['f1_measure'])
# Render evaluation results
if switch['log_image'] or switch['log_sound']:
sub_fac = int(model_params['audio_sr'] / switch['midi_sr'])
est = roll_subsample(est.T, sub_fac)
ref = roll_subsample(ref.T, sub_fac)
if switch['log_image']:
write_images(est, ref, switch['midi_sr'], args.threshold,
(8, 6), images_summary, images_buffer, writer,
step, sess)
if switch['log_sound']:
write_audio(est, ref, switch['midi_sr'],
model_params['audio_sr'], 0.007, audio_summary,
audio_data, writer, step, sess)
except KeyboardInterrupt:
# Introduce a line break after ^C is displayed so save message
# is on its own line.
print()
finally:
if step > last_saved_step:
save(saver, sess, logdir, step)
coord.request_stop()
coord.join(threads)
flush_n_close(writer, sess)
def generate_best(dir_list, border=4):
img_list_dict = dict()
# get sorted img names in corresponding model
for dir_name in dir_list:
img_list_dict[dir_name] = sorted(os.listdir(dir_name))
length = len(dir_list)
for i in range(0, 100): # for every Urban img
# load hr img
hr_basename = img_list_dict[dir_list[0]][i]
hr_path = osp.join(dir_list[0], hr_basename)
hr_img = imageio.imread(hr_path, pilmode='RGB')
h, w = hr_img.shape[:-1]
h_step, w_step = h // 20, w // 20
img_psnrs, img_ssims = [], []
# get metrics of different models for this img
for k in range(length-1):
basename = img_list_dict[dir_list[k+1]][i]
path = osp.join(dir_list[k+1], basename)
img = imageio.imread(path, pilmode='RGB')
if dir_list[k+1] == 'IDN':
img = cv2.copyMakeBorder(img, 4, 4, 4, 4, cv2.BORDER_REPLICATE)
psnr, ssim = calc_metrics(hr_img, img, crop_border=border, test_Y=True)
img_psnrs.append(psnr)
img_ssims.append(ssim)
str_img_psnrs = ['{:.2f}'.format(x) for x in img_psnrs]
print('full img[{:03d}] | {}'.format((i+1), str_img_psnrs))
# whether best is ours
if np.argmax(np.array(img_psnrs)) < length-2 or np.argmax(np.array(img_ssims)) < length-2:
continue
# fixed stride for different location, get 64*64*3 patch
for y in range(0, h-64, h_step):
for x in range(0, w-64, w_step):
imgs, psnrs, ssims = [], [], []
# plot rectangle on hr img
hr_img1 = hr_img.copy()
cv2.rectangle(hr_img1, (x, y), (x+63, y+63), (255, 0, 0), 2)
left, right, top, bottom = get_direction(x+32, y+32, w, h)
imgs.append(hr_img1[top:bottom+1, left:right+1, :]) # 513 * 513 * 3
hr_patch = hr_img[y:y+64, x:x+64, :]
imgs.append(hr_patch)
# for different model, get corresponding patch
for k in range(length-1):
basename = img_list_dict[dir_list[k+1]][i]
path = osp.join(dir_list[k+1], basename)
img = imageio.imread(path, pilmode='RGB')
if dir_list[k+1] == 'IDN':
img = cv2.copyMakeBorder(img, 4, 4, 4, 4, cv2.BORDER_REPLICATE)
img_patch = img[y:y+64, x:x+64, :]
imgs.append(img_patch)
# calculate psnr and ssim
psnr, ssim = calc_metrics(hr_patch, img_patch)
psnrs.append(psnr)
ssims.append(ssim)
str_psnrs = ['{:.2f}'.format(psnr) for psnr in psnrs]
print('[{:03d}] | ({}/{}, {}/{}) | {}'.format((i+1), y, h, x, w, str_psnrs))
if np.argmax(np.array(psnrs)) == length-2 and np.argmax(np.array(ssims)) == length-2:
print('Saving...')
plot_compare(imgs, img_psnrs, img_ssims, i+1, '{}_{}'.format(y, x), dir_list)
def train(train_loader, model, criterion, optimizer, epoch, device):
batch_time = AverageMeter()
data_time = AverageMeter()
model.train()
train_len = len(train_loader)
scores = {
'acc': 0,
'rec': 0,
'f1': 0,
'hamm': 0,
'emr': 0,
}
end = time.time()
for batch_idx, (data, target) in enumerate(train_loader):
data_time.update(time.time() - end)
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
act_output, reconstructions = model(data)
act_loss, reconstruction_loss, total_loss = criterion(
act_output, target, data, reconstructions)
acc, rec, f1, hamm, emr = calc_metrics(act_output, target)
scores.update(acc=scores['acc'] + acc,
rec=scores['rec'] + rec,
f1=scores['f1'] + f1,
hamm=scores['hamm'] + hamm,
emr=scores['emr'] + emr)
global_step = (batch_idx + 1) + (epoch - 1) * len(train_loader)
# change the learning rate exponentially
exp_lr_decay(optimizer=optimizer, global_step=global_step)
total_loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % args.log_interval == 0:
# Logging
train_writer.add_scalar('Loss/BCE', act_loss.item(), global_step)
if args.add_decoder:
train_writer.add_scalar('Loss/Reconstruction',
reconstruction_loss.item(),
global_step)
log_reconstruction_sample(train_writer, data, reconstructions,
global_step)
train_writer.add_scalar('Loss/Total', total_loss.item(),
global_step)
train_writer.add_scalar('Metrics/Precision', acc, global_step)
train_writer.add_scalar('Metrics/Recall', rec, global_step)
train_writer.add_scalar('Metrics/F1-score', f1, global_step)
train_writer.add_scalar('Metrics/HammingScore', hamm, global_step)
train_writer.add_scalar('Metrics/ExactMatchRatio', emr,
global_step)
# Console output
print('Train Epoch: {}\t[{}/{} ({:.0f}%)]\t'
'Loss: {:.6f}\tF1-Score: {:.6f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})'.format(
epoch,
batch_idx * len(data),
len(train_loader.dataset),
100. * batch_idx / len(train_loader),
total_loss.item(),
f1,
batch_time=batch_time,
data_time=data_time))
scores.update(acc=scores['acc'] / train_len,
rec=scores['rec'] / train_len,
f1=scores['f1'] / train_len,
hamm=scores['hamm'] / train_len,
emr=scores['emr'] / train_len)
return scores
def forward(self,
data_batch,
epoch,
use_second_order,
use_multi_step_loss_optimization,
num_steps,
training_phase,
do_evaluation=False):
"""
Runs a forward outer loop pass on the batch of tasks using the MAML/++ framework.
:param data_batch: A data batch containing the support and target sets.
:param epoch: Current epoch's index
:param use_second_order: A boolean saying whether to use second order derivatives.
:param use_multi_step_loss_optimization: Whether to optimize on the outer loop using just the last step's
target loss (True) or whether to use multi step loss which improves the stability of the system (False)
:param num_steps: Number of inner loop steps.
:param training_phase: Whether this is a training phase (True) or an evaluation phase (False)
:return: A dictionary with the collected losses of the current outer forward propagation.
"""
frames = data_batch
total_losses = []
loss_accumulator = {'total': utils.AverageMeter()}
metrics = {'psnr': utils.AverageMeter(), 'ssim': utils.AverageMeter()}
per_task_target_preds = [[] for i in range(len(frames[0]))]
self.net.zero_grad()
for task_id in range(len(frames[0])): # loop over batch dimension
task_losses = []
per_step_loss_importance_vectors = self.get_per_step_loss_importance_vector(
)
names_weights_copy = self.get_inner_loop_parameter_dict(
self.net.named_parameters())
names_weights_copy = {
name.replace('module.', ''): value
for name, value in names_weights_copy.items()
}
# inner loop
support_idxs = self.support_idxs # frame indices: input[0, 2, 4, 6] --> output[3]
target_idx = self.target_idxs
self.inner_loop_optimizer.initialize_state()
# Attenuate the initialization for L2F
if self.args.attenuate:
task_embeddings = self.get_task_embeddings(
frames, task_id, names_weights_copy)
names_weights_copy = self.attenuate_init(
task_embeddings=task_embeddings,
names_weights_copy=names_weights_copy)
for num_step in range(num_steps):
support_loss = 0
for ind in support_idxs:
_loss, _ = self.net_forward(
frame0=frames[ind[0]][task_id].unsqueeze(0),
frame1=frames[ind[2]][task_id].unsqueeze(0),
target=frames[ind[1]][task_id].unsqueeze(0),
weights=names_weights_copy,
backup_running_statistics=True if
(num_step == 0) else False,
training=True,
num_step=num_step)
support_loss = support_loss + _loss['total']
names_weights_copy = self.apply_inner_loop_update(
loss=support_loss,
names_weights_copy=names_weights_copy,
use_second_order=use_second_order,
current_step_idx=num_step)
kwargs = {
'backup_running_statistics': False,
'training': True,
'num_step': num_step
}
if use_multi_step_loss_optimization and training_phase and epoch < self.args.multi_step_loss_num_epochs:
target_loss, target_preds = self.net_forward(
frame0=frames[target_idx[0]][task_id].unsqueeze(0),
frame1=frames[target_idx[2]][task_id].unsqueeze(0),
target=frames[target_idx[1]][task_id].unsqueeze(0),
weights=names_weights_copy,
**kwargs)
task_losses.append(
per_step_loss_importance_vectors[num_step] *
target_loss['total'])
self.update_loss_metrics(loss_accumulator, target_loss)
if not training_phase:
kwargs = {
'backup_running_statistics': False,
'training': True,
'num_step': num_steps
}
with torch.no_grad():
target_loss, target_preds = self.net_forward(
frame0=frames[target_idx[0]][task_id].unsqueeze(0),
frame1=frames[target_idx[2]][task_id].unsqueeze(0),
target=frames[target_idx[1]][task_id].unsqueeze(0),
weights=names_weights_copy,
**kwargs)
task_losses.append(target_loss['total'])
self.update_loss_metrics(loss_accumulator, target_loss)
elif not (use_multi_step_loss_optimization and training_phase
and epoch < self.args.multi_step_loss_num_epochs):
kwargs = {
'backup_running_statistics': False,
'training': True,
'num_step': num_steps
}
target_loss, target_preds = self.net_forward(
frame0=frames[target_idx[0]][task_id].unsqueeze(0),
frame1=frames[target_idx[2]][task_id].unsqueeze(0),
target=frames[target_idx[1]][task_id].unsqueeze(0),
weights=names_weights_copy,
**kwargs)
task_losses.append(target_loss['total'])
self.update_loss_metrics(loss_accumulator, target_loss)
if self.args.model == 'superslomo':
per_task_target_preds[task_id] = self.revNormalize(
target_preds.detach().squeeze(0)).unsqueeze(0)
elif self.args.model == 'voxelflow':
per_task_target_preds[task_id] = (
(target_preds.detach().squeeze(0) * self.std + self.mean) /
255.0).unsqueeze(0)
else:
per_task_target_preds[task_id] = target_preds.detach(
) # target_preds.shape: (1, C, H, W)
if do_evaluation:
if self.args.model == 'superslomo':
output = self.revNormalize(target_preds.squeeze(0))
target = self.revNormalize(frames[target_idx[1]][task_id])
elif self.args.model == 'voxelflow':
output = (target_preds.squeeze(0) * self.std +
self.mean) / 255.0
target = (frames[target_idx[1]][task_id] * self.std +
self.mean) / 255.0
else:
output = target_preds.squeeze(0)
target = frames[target_idx[1]][task_id]
output = output.detach()
target = target.detach()
psnr, ssim = utils.calc_metrics(output, target)
# print(psnr, ssim)
metrics['psnr'].update(psnr)
metrics['ssim'].update(ssim)
else:
pass
task_losses = torch.sum(torch.stack(task_losses))
total_losses.append(task_losses)
if not training_phase:
self.net.restore_backup_stats()
losses = self.get_across_task_loss_metrics(
total_losses=total_losses, specific_losses=loss_accumulator)
for idx, item in enumerate(per_step_loss_importance_vectors):
losses['loss_importance_vector_{}'.format(
idx)] = item.detach().cpu().numpy()
return losses, per_task_target_preds, metrics
def validate(valid_loader, model, criterion, epoch, cfg):
losses = {
'softmax': AverageMeter(),
'center': AverageMeter(),
'total': AverageMeter()
}
accs = AverageMeter()
y_pred, y_true, y_scores = [], [], []
# switch to evaluate mode
model.eval()
with tqdm(total=int(len(valid_loader.dataset) /
cfg['batch_size'])) as pbar:
with torch.no_grad():
for i, (images, target) in enumerate(valid_loader):
images = images.to(device)
target = target.to(device)
# compute output
feat, output, A = model(images)
l_softmax = criterion['softmax'](output, target)
l_center = criterion['center'](feat, A, target)
l_total = l_softmax + cfg['lamb'] * l_center
# measure accuracy and record loss
acc, pred = accuracy(output, target)
losses['softmax'].update(l_softmax.item(), images.size(0))
losses['center'].update(l_center.item(), images.size(0))
losses['total'].update(l_total.item(), images.size(0))
accs.update(acc.item(), images.size(0))
# collect for metrics
y_pred.append(pred)
y_true.append(target)
y_scores.append(output.data)
# progressbar
pbar.set_description(
f'VALIDATING [{epoch:03d}/{cfg["epochs"]}]')
pbar.update(1)
metrics = calc_metrics(y_pred, y_true, y_scores)
progress = (f'[-] VALID [{epoch:03d}/{cfg["epochs"]}] | '
f'L={losses["total"].avg:.4f} | '
f'Ls={losses["softmax"].avg:.4f} | '
f'Lsc={losses["center"].avg:.4f} | '
f'acc={accs.avg:.4f} | '
f'rec={metrics["rec"]:.4f} | '
f'f1={metrics["f1"]:.4f} | '
f'aucpr={metrics["aucpr"]:.4f} | '
f'aucroc={metrics["aucroc"]:.4f}')
print(progress)
# save model checkpoints for best valid
if accs.avg > best_valid['acc']:
save_checkpoint(epoch, model, cfg, tag='best_valid_acc.pth')
if metrics['rec'] > best_valid['rec']:
save_checkpoint(epoch, model, cfg, tag='best_valid_rec.pth')
best_valid['acc'] = max(best_valid['acc'], accs.avg)
best_valid['rec'] = max(best_valid['rec'], metrics['rec'])
best_valid['f1'] = max(best_valid['f1'], metrics['f1'])
best_valid['aucpr'] = max(best_valid['aucpr'], metrics['aucpr'])
best_valid['aucroc'] = max(best_valid['aucroc'], metrics['aucroc'])
write_log(losses, accs.avg, metrics, epoch, tag='valid')
def _run_VI(self, fixed, moving, var_params_q_v):
self.__init_optimizer_q_v(var_params_q_v)
data_loss, reg_loss, entropy_loss = self.losses['data'][
'loss'], self.losses['reg']['loss'], self.losses['entropy']
# .nii.gz/.vtk
with torch.no_grad():
save_fixed_im(self.save_dirs, self.im_spacing, fixed['im'])
save_fixed_mask(self.save_dirs, self.im_spacing, fixed['mask'])
save_moving_im(self.save_dirs, self.im_spacing, moving['im'])
save_moving_mask(self.save_dirs, self.im_spacing, moving['mask'])
for iter_no in range(self.start_iter_VI, self.no_iters_VI + 1):
# needed to calculate the maximum update in terms of the L2 norm
var_params_q_v_prev = {
k: v.detach().clone()
for k, v in var_params_q_v.items()
}
v_sample1_unsmoothed, v_sample2_unsmoothed = sample_q_v(
var_params_q_v, no_samples=2)
loss_terms1, output, aux = self.__calc_sample_loss_VI(
data_loss, reg_loss, entropy_loss, fixed, moving,
var_params_q_v, v_sample1_unsmoothed)
loss_terms2, _, _ = self.__calc_sample_loss_VI(
data_loss, reg_loss, entropy_loss, fixed, moving,
var_params_q_v, v_sample2_unsmoothed)
# data
data_term = (loss_terms1['data'] + loss_terms2['data']) / 2.0
data_term -= self.losses['data']['scale_prior'](
data_loss.log_scales).sum()
data_term -= self.losses['data']['proportion_prior'](
data_loss.log_proportions).sum()
# regularisation
reg_term = (loss_terms1['reg'] + loss_terms2['reg']) / 2.0
if reg_loss.learnable:
if reg_loss.__class__.__name__ == 'RegLoss_LogNormal':
reg_term -= (loss_terms1['reg_loc_prior'] +
loss_terms2['reg_loc_prior']) / 2.0
reg_term -= self.losses['reg']['scale_prior'](
reg_loss.log_scale).sum()
elif reg_loss.__class__.__name__ == 'RegLoss_L2':
reg_term -= (loss_terms1['w_reg_prior'] +
loss_terms2['w_reg_prior']) / 2.0
# entropy
entropy_term = (loss_terms1['entropy'] +
loss_terms2['entropy']) / 2.0
entropy_term += entropy_loss(log_var=var_params_q_v['log_var'],
u=var_params_q_v['u']).sum()
# total loss
loss = data_term + reg_term - entropy_term
if reg_loss.learnable:
self.optimizer_reg.zero_grad()
self.optimizer_q_v.zero_grad()
loss.backward() # backprop
if reg_loss.learnable:
self.optimizer_reg.step()
self.optimizer_q_v.step()
"""
tensorboard and logging
"""
with torch.no_grad():
self.writer.set_step(iter_no)
# model parameters
for idx in range(data_loss.no_components):
self.metrics.update(f'VI/train/GMM/scale_{idx}',
data_loss.scales[idx].item())
self.metrics.update(f'VI/train/GMM/proportion_{idx}',
data_loss.proportions[idx].item())
if reg_loss.learnable:
if reg_loss.__class__.__name__ == 'RegLoss_LogNormal':
self.metrics.update('VI/train/reg/loc',
reg_loss.loc.item())
self.metrics.update('VI/train/reg/scale',
reg_loss.scale.item())
elif reg_loss.__class__.__name__ == 'RegLoss_L2':
self.metrics.update('VI/train/reg/w_reg',
reg_loss.log_w_reg.exp().item())
if self.virutal_decimation:
self.metrics.update('VI/train/VD/alpha',
aux['alpha'].item())
# losses
self.metrics.update('VI/train/data_term', data_term.item())
self.metrics.update('VI/train/reg_term', reg_term.item())
self.metrics.update('VI/train/entropy_term',
entropy_term.item())
self.metrics.update('VI/train/total_loss', loss.item())
# other
self.metrics.update('VI/train/reg/energy',
aux['reg_energy'].item())
self.metrics.update('VI/train/no_non_diffeomorphic_voxels',
aux['no_non_diffeomorphic_voxels'].item())
for key in var_params_q_v:
max_update, max_update_idx = max_field_update(
var_params_q_v_prev[key], var_params_q_v[key])
self.metrics.update(f'VI/train/max_updates/{key}',
max_update.item())
if iter_no % self.log_period_VI == 0 or iter_no == self.no_iters_VI:
# metrics
seg_moving_warped = self.registration_module(
moving['seg'], output['transformation'])
ASD, DSC = calc_metrics(fixed['seg'], seg_moving_warped,
self.structures_dict,
self.im_spacing)
for structure_idx, structure in enumerate(
self.structures_dict):
ASD_val, DSC_val = ASD[0][structure_idx], DSC[0][
structure_idx]
self.metrics.update(f'VI/train/ASD/{structure}',
ASD_val)
self.metrics.update(f'VI/train/DSC/{structure}',
DSC_val)
# visualisation in tensorboard
var_params_q_v_smoothed = self.__get_var_params_smoothed(
var_params_q_v)
log_hist_res(self.writer,
aux['residuals'],
data_loss,
model='VI')
log_images(self.writer, fixed['im'], moving['im'],
output['im_moving_warped'])
log_fields(self.writer, var_params_q_v_smoothed,
output['displacement'], output['log_det_J'])
def evaluate(self, loader):
print('Evaluating at {} epochs...'.format(self.epoch))
torch.set_grad_enabled(False)
# remove previous viz results
makedirs(self.args.vis, remove=True)
self.netwrapper.eval()
# initialize meters
loss_meter = AverageMeter()
sdr_mix_meter = AverageMeter()
sdr_meter = AverageMeter()
sir_meter = AverageMeter()
sar_meter = AverageMeter()
# initialize HTML header
visualizer = HTMLVisualizer(os.path.join(self.args.vis, 'index.html'))
header = ['Filename', 'Input Mixed Audio']
for n in range(1, self.args.num_mix + 1):
header += [
'Video {:d}'.format(n), 'Predicted Audio {:d}'.format(n),
'GroundTruth Audio {}'.format(n),
'Predicted Mask {}'.format(n), 'GroundTruth Mask {}'.format(n)
]
header += ['Loss weighting']
visualizer.add_header(header)
vis_rows = []
eval_num = 0
valid_num = 0
#for i, batch_data in enumerate(self.loader['eval']):
for i, batch_data in enumerate(loader):
# forward pass
eval_num += batch_data['mag_mix'].shape[0]
with torch.no_grad():
err, outputs = self.netwrapper.forward(batch_data, args)
err = err.mean()
if self.mode == 'train':
self.writer.add_scalar('data/val_loss', err,
self.args.epoch_iters * self.epoch + i)
loss_meter.update(err.item())
print('[Eval] iter {}, loss: {:.4f}'.format(i, err.item()))
# calculate metrics
sdr_mix, sdr, sir, sar, cur_valid_num = calc_metrics(
batch_data, outputs, self.args)
print("sdr_mix, sdr, sir, sar: ", sdr_mix, sdr, sir, sar)
sdr_mix_meter.update(sdr_mix)
sdr_meter.update(sdr)
sir_meter.update(sir)
sar_meter.update(sar)
valid_num += cur_valid_num
'''
# output visualization
if len(vis_rows) < self.args.num_vis:
output_visuals(vis_rows, batch_data, outputs, self.args)
'''
metric_output = '[Eval Summary] Epoch: {}, Loss: {:.4f}, ' \
'SDR_mixture: {:.4f}, SDR: {:.4f}, SIR: {:.4f}, SAR: {:.4f}'.format(
self.epoch, loss_meter.average(),
sdr_mix_meter.sum_value()/eval_num,
sdr_meter.sum_value()/eval_num,
sir_meter.sum_value()/eval_num,
sar_meter.sum_value()/eval_num
)
if valid_num / eval_num < 0.8:
metric_output += ' ---- Invalid ---- '
print(metric_output)
learning_rate = ' lr_sound: {}, lr_frame: {}'.format(
self.args.lr_sound, self.args.lr_frame)
with open(self.args.log, 'a') as F:
F.write(metric_output + learning_rate + '\n')
self.history['val']['epoch'].append(self.epoch)
self.history['val']['err'].append(loss_meter.average())
self.history['val']['sdr'].append(sdr_meter.sum_value() / eval_num)
self.history['val']['sir'].append(sir_meter.sum_value() / eval_num)
self.history['val']['sar'].append(sar_meter.sum_value() / eval_num)
'''
print('Plotting html for visualization...')
visualizer.add_rows(vis_rows)
visualizer.write_html()
'''
# Plot figure
if self.epoch > 0:
print('Plotting figures...')
plot_loss_metrics(self.args.ckpt, self.history)
def evaluation_iteration(self, val_sample, total_losses, pbar_val, phase):
"""
Runs a validation iteration, updates the progress bar and returns the total and current epoch val losses.
:param val_sample: A sample from the data provider
:param total_losses: The current total losses dictionary to be updated.
:param pbar_val: The progress bar of the val stage.
:return: The updated val_losses, total_losses
"""
images, metadata = val_sample
def _eval_iter(frames):
H, W = frames[0].shape[-2:]
if H * W > 5e5 or (
self.args.model == 'rrin' and H * W > 3e5
): # or (self.args.model == 'dain' and H * W > 1e5):
print(H, W)
if H > W:
images_0 = [im[:, :, :H // 2, :] for im in frames]
images_1 = [im[:, :, H // 2:, :] for im in frames]
else:
images_0 = [im[:, :, :, :W // 2] for im in frames]
images_1 = [im[:, :, :, W // 2:] for im in frames]
losses_0, outputs_0, metrics_0 = _eval_iter(images_0)
losses_1, outputs_1, metrics_1 = _eval_iter(images_1)
outputs = [
torch.cat([outputs_0[i], outputs_1[i]],
dim=2 if H > W else 3)
for i in range(len(outputs_0))
]
losses = losses_0
for k, v in losses_1.items():
losses[k] = (v + losses[k]) / 2
if k == 'loss':
losses[k] = losses[k].detach()
#metrics = metrics_0
#for k, v in metrics_1.items():
# metrics[k].update(val=v.avg, n=v.count)
del losses_0, losses_1, outputs_0, outputs_1 #, metrics_0, metrics_1
else:
losses, outputs, metrics = self.model.run_validation_iter(
data_batch=frames)
losses['loss'] = losses['loss'].detach()
return losses, outputs, None #metrics
losses, outputs, _ = _eval_iter(images)
output = outputs[0].squeeze(0).detach()
target = images[3][0].detach().cuda()
if self.args.model == 'voxelflow':
target = (target * self.model.std + self.model.mean) / 255.0
elif self.args.model == 'superslomo':
target = self.model.revNormalize(target)
metrics = {'psnr': utils.AverageMeter(), 'ssim': utils.AverageMeter()}
psnr, ssim = utils.calc_metrics(output, target)
metrics['psnr'].update(psnr)
metrics['ssim'].update(ssim)
val_output_update = self.build_loss_summary_string(losses, metrics)
pbar_val.update(1)
pbar_val.set_description("val_phase {} -> {}".format(
self.epoch, val_output_update))
return losses, outputs, metrics
def main():
args = get_arguments()
with open(args.model_params, 'r') as f:
model_params = json.load(f)
with open(args.training_params, 'r') as f:
train_params = json.load(f)
try:
directories = validate_directories(args)
except ValueError as e:
print('Some arguments are wrong:')
print(str(e))
return
logdir = directories['logdir']
restore_from = directories['restore_from']
# Even if we restored the model, we will treat it as new training
# if the trained model is written into an arbitrary location.
is_overwritten_training = logdir != restore_from
receptive_field = WaveNetModel.calculate_receptive_field(
model_params['filter_width'],
model_params['dilations'],
model_params['initial_filter_width'])
# Save arguments and model params into file
save_run_config(args, receptive_field, STARTED_DATESTRING, logdir)
# Create coordinator.
coord = tf.train.Coordinator()
# Create data loader.
with tf.name_scope('create_inputs'):
reader = WavMidReader(data_dir=args.data_dir_train,
coord=coord,
audio_sample_rate=model_params['audio_sr'],
receptive_field=receptive_field,
velocity=args.velocity,
sample_size=args.sample_size,
queues_size=(10, 10*args.batch_size))
data_batch = reader.dequeue(args.batch_size)
# Create model.
net = WaveNetModel(
batch_size=args.batch_size,
dilations=model_params['dilations'],
filter_width=model_params['filter_width'],
residual_channels=model_params['residual_channels'],
dilation_channels=model_params['dilation_channels'],
skip_channels=model_params['skip_channels'],
output_channels=model_params['output_channels'],
use_biases=model_params['use_biases'],
initial_filter_width=model_params['initial_filter_width'])
input_data = tf.placeholder(dtype=tf.float32,
shape=(args.batch_size, None, 1))
input_labels = tf.placeholder(dtype=tf.float32,
shape=(args.batch_size, None,
model_params['output_channels']))
loss, probs = net.loss(input_data=input_data,
input_labels=input_labels,
pos_weight=train_params['pos_weight'],
l2_reg_str=train_params['l2_reg_str'])
optimizer = optimizer_factory[args.optimizer](
learning_rate=train_params['learning_rate'],
momentum=train_params['momentum'])
trainable = tf.trainable_variables()
optim = optimizer.minimize(loss, var_list=trainable)
# Set up logging for TensorBoard.
writer = tf.summary.FileWriter(logdir)
writer.add_graph(tf.get_default_graph())
run_metadata = tf.RunMetadata()
summaries = tf.summary.merge_all()
histograms = tf.summary.merge_all(key=HKEY)
# Separate summary ops for validation, since they are
# calculated only once per evaluation cycle.
with tf.name_scope('validation_summaries'):
metric_summaries = metrics_empty_dict()
metric_value = tf.placeholder(tf.float32)
for name in metric_summaries.keys():
metric_summaries[name] = tf.summary.scalar(name, metric_value)
images_buffer = tf.placeholder(tf.string)
images_batch = tf.stack(
[tf.image.decode_png(images_buffer[0], channels=4),
tf.image.decode_png(images_buffer[1], channels=4),
tf.image.decode_png(images_buffer[2], channels=4)])
images_summary = tf.summary.image('estim', images_batch)
audio_data = tf.placeholder(tf.float32)
audio_summary = tf.summary.audio('input', audio_data,
model_params['audio_sr'])
# Set up session
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.trainable_variables(),
max_to_keep=args.max_checkpoints)
# Trainer for keeping best validation-performing model
# and optional early stopping.
trainer = Trainer(sess, logdir, train_params['early_stop_limit'], 0.999)
try:
saved_global_step = load(saver, sess, restore_from)
if is_overwritten_training or saved_global_step is None:
# The first training step will be saved_global_step + 1,
# therefore we put -1 here for new or overwritten trainings.
saved_global_step = -1
except:
print('Something went wrong while restoring checkpoint. '
'Training will be terminated to avoid accidentally '
'overwriting the previous model.')
raise
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
reader.start_threads(sess)
step = None
last_saved_step = saved_global_step
try:
for step in range(saved_global_step + 1, train_params['num_steps']):
waveform, pianoroll = sess.run([data_batch[0], data_batch[1]])
feed_dict = {input_data : waveform, input_labels : pianoroll}
# Reload switches from file on each step
with open(RUNTIME_SWITCHES, 'r') as f:
switch = json.load(f)
start_time = time.time()
if switch['store_meta'] and step % switch['store_every'] == 0:
# Slow run that stores extra information for debugging.
print('Storing metadata')
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
summary, loss_value, _ = sess.run(
[summaries, loss, optim],
feed_dict=feed_dict,
options=run_options,
run_metadata=run_metadata)
writer.add_summary(summary, step)
writer.add_run_metadata(run_metadata,
'step_{:04d}'.format(step))
tl = timeline.Timeline(run_metadata.step_stats)
timeline_path = os.path.join(logdir, 'timeline.trace')
with open(timeline_path, 'w') as f:
f.write(tl.generate_chrome_trace_format(show_memory=True))
else:
summary, loss_value, _ = sess.run([summaries, loss, optim],
feed_dict=feed_dict)
writer.add_summary(summary, step)
duration = time.time() - start_time
print('step {:d} - loss = {:.3f}, ({:.3f} sec/step)'
.format(step, loss_value, duration))
if step % switch['checkpoint_every'] == 0:
save(saver, sess, logdir, step)
last_saved_step = step
# Evaluate model performance on validation data
if step % switch['evaluate_every'] == 0:
if switch['histograms']:
hist_summary = sess.run(histograms)
writer.add_summary(hist_summary, step)
print('evaluating...')
stats = 0, 0, 0, 0, 0, 0
est = np.empty([0, model_params['output_channels']])
ref = np.empty([0, model_params['output_channels']])
b_data, b_labels, b_cntr = (
np.empty((0, args.sample_size + receptive_field - 1, 1)),
np.empty((0, model_params['output_channels'])),
args.batch_size)
# if (batch_size * sample_size > valid_data) single_pass() again
while est.size == 0: # and ref.size == 0 and sum(stats) == 0 ...
for data, labels in reader.single_pass(
sess, args.data_dir_valid):
# cumulate batch
if b_cntr > 1:
b_data, b_labels, decr = cumulateBatch(
data, labels, b_data, b_labels)
b_cntr -= decr
continue
elif args.batch_size > 1:
b_data, b_labels, decr = cumulateBatch(
data, labels, b_data, b_labels)
if not decr:
continue
data = b_data
labels = b_labels
# reset batch cumulation variables
b_data, b_labels, b_cntr = (
np.empty((
0, args.sample_size + receptive_field - 1, 1
)),
np.empty((0, model_params['output_channels'])),
args.batch_size)
predictions = sess.run(
probs, feed_dict={input_data : data})
# Aggregate sums for metrics calculation
stats_chunk = calc_stats(
predictions, labels, args.threshold)
stats = tuple([sum(x) for x in zip(stats, stats_chunk)])
est = np.append(est, predictions, axis=0)
ref = np.append(ref, labels, axis=0)
metrics = calc_metrics(None, None, None, stats=stats)
write_metrics(metrics, metric_summaries, metric_value,
writer, step, sess)
trainer.check(metrics['f1_measure'])
# Render evaluation results
if switch['log_image'] or switch['log_sound']:
sub_fac = int(model_params['audio_sr']/switch['midi_sr'])
est = roll_subsample(est.T, sub_fac)
ref = roll_subsample(ref.T, sub_fac)
if switch['log_image']:
write_images(est, ref, switch['midi_sr'], args.threshold,
(8, 6), images_summary, images_buffer,
writer, step, sess)
if switch['log_sound']:
write_audio(est, ref, switch['midi_sr'],
model_params['audio_sr'], 0.007,
audio_summary, audio_data,
writer, step, sess)
except KeyboardInterrupt:
# Introduce a line break after ^C is displayed so save message
# is on its own line.
print()
finally:
if step > last_saved_step:
save(saver, sess, logdir, step)
coord.request_stop()
coord.join(threads)
flush_n_close(writer, sess)
def train(train_loader, model, criterion, optimizer, epoch, cfg):
losses = {
'softmax': AverageMeter(),
'center': AverageMeter(),
'total': AverageMeter()
}
accs = AverageMeter()
y_pred, y_true, y_scores = [], [], []
# switch to train mode
model.train()
with tqdm(total=int(len(train_loader.dataset) /
cfg['batch_size'])) as pbar:
for i, (images, target) in enumerate(train_loader):
images = images.to(device)
target = target.to(device)
# compute output
feat, output, A = model(images)
l_softmax = criterion['softmax'](output, target)
l_center = criterion['center'](feat, A, target)
l_total = l_softmax + cfg['lamb'] * l_center
# measure accuracy and record loss
acc, pred = accuracy(output, target)
losses['softmax'].update(l_softmax.item(), images.size(0))
losses['center'].update(l_center.item(), images.size(0))
losses['total'].update(l_total.item(), images.size(0))
accs.update(acc.item(), images.size(0))
# collect for metrics
y_pred.append(pred)
y_true.append(target)
y_scores.append(output.data)
# compute grads + opt step
optimizer['softmax'].zero_grad()
optimizer['center'].zero_grad()
l_total.backward()
optimizer['softmax'].step()
optimizer['center'].step()
# progressbar
pbar.set_description(f'TRAINING [{epoch:03d}/{cfg["epochs"]}]')
pbar.set_postfix({
'L': losses["total"].avg,
'Ls': losses["softmax"].avg,
'Lsc': losses["center"].avg,
'acc': accs.avg
})
pbar.update(1)
metrics = calc_metrics(y_pred, y_true, y_scores)
progress = (f'[-] TRAIN [{epoch:03d}/{cfg["epochs"]}] | '
f'L={losses["total"].avg:.4f} | '
f'Ls={losses["softmax"].avg:.4f} | '
f'Lsc={losses["center"].avg:.4f} | '
f'acc={accs.avg:.4f} | '
f'rec={metrics["rec"]:.4f} | '
f'f1={metrics["f1"]:.4f} | '
f'aucpr={metrics["aucpr"]:.4f} | '
f'aucroc={metrics["aucroc"]:.4f}')
print(progress)
write_log(losses, accs.avg, metrics, epoch, tag='train')
def _run_MCMC(self, fixed, moving, var_params_q_v):
data_loss, reg_loss = self.losses['data']['loss'], self.losses['reg'][
'loss']
fixed = {
k: v.expand(self.no_chains, *v.shape[1:])
for k, v in fixed.items()
}
moving = {
k: v.expand(self.no_chains, *v.shape[1:])
for k, v in moving.items()
}
self.__SGLD_init(var_params_q_v)
no_samples = self.no_chains * self.no_samples_MCMC // self.log_period_MCMC
samples = torch.zeros([no_samples, 3,
*self.dims]) # samples used in the evaluation
test_sample_idx = 0
self.logger.info(f'\nNO. CHAINS: {self.no_chains}, BURNING IN...')
for sample_no in range(
1, self.no_iters_burn_in + self.no_samples_MCMC + 1):
if sample_no < self.no_iters_burn_in and sample_no % self.log_period_MCMC == 0:
self.logger.info(
f'burn-in sample no. {sample_no}/{self.no_iters_burn_in}')
"""
stochastic gradient Langevin dynamics
"""
loss_terms, output, aux = self._SGLD_transition(
fixed, moving, data_loss, reg_loss)
if sample_no == self.no_iters_burn_in:
self.logger.info('ENDED BURNING IN')
"""
outputs
"""
with torch.no_grad():
self.writer.set_step(sample_no)
if self.no_samples_MCMC < 1e4 or (
sample_no - 1
) % 100 == 0: # NOTE (DG): the logs are too large otherwise
# model parameters
for idx in range(data_loss.no_components):
self.metrics.update(f'MCMC/GMM/scale_{idx}',
data_loss.scales[idx].item())
self.metrics.update(f'MCMC/GMM/proportion_{idx}',
data_loss.proportions[idx].item())
if reg_loss.learnable:
if reg_loss.__class__.__name__ == 'RegLoss_LogNormal':
self.metrics.update('MCMC/reg/loc',
reg_loss.loc.item())
self.metrics.update('MCMC/reg/scale',
reg_loss.scale.item())
elif reg_loss.__class__.__name__ == 'RegLoss_L2':
self.metrics.update(
'MCMC/reg/w_reg',
reg_loss.log_w_reg.exp().item())
# losses
total_loss = sum(loss_terms['data']) + sum(
loss_terms['reg'])
self.metrics.update(f'MCMC/avg_loss',
total_loss.item() / self.no_chains)
for idx in range(self.no_chains):
self.metrics.update(f'MCMC/chain_{idx}/data_term',
loss_terms['data'][idx].item())
self.metrics.update(f'MCMC/chain_{idx}/reg_term',
loss_terms['reg'][idx].item())
self.metrics.update(f'MCMC/chain_{idx}/VD/alpha',
aux['alpha'][idx].item())
self.metrics.update(f'MCMC/chain_{idx}/reg/energy',
aux['reg_energy'][idx].item())
if sample_no > self.no_iters_burn_in:
if sample_no % self.log_period_MCMC == 0 or sample_no == self.no_samples_MCMC:
self.writer.set_step(sample_no - self.no_iters_burn_in)
displacement, transformation = output[
'displacement'], output['transformation']
im_moving_warped = output['im_moving_warped']
seg_moving_warped = self.registration_module(
moving['seg'], transformation)
ASD, DSC = calc_metrics(fixed['seg'],
seg_moving_warped,
self.structures_dict,
self.im_spacing,
no_samples=self.no_chains)
no_non_diffeomorphic_voxels, log_det_J = calc_no_non_diffeomorphic_voxels(
transformation, self.diff_op)
v_curr_state_smoothed = output['curr_state']
v_norm, displacement_norm = calc_norm(
v_curr_state_smoothed), calc_norm(displacement)
for idx in range(self.no_chains):
samples[test_sample_idx] = displacement[
idx].detach().cpu()
test_sample_idx += 1
for structure_idx, structure in enumerate(
self.structures_dict):
ASD_val, DSC_val = ASD[idx][
structure_idx], DSC[idx][structure_idx]
self.metrics.update(
f'MCMC/chain_{idx}/ASD/{structure}',
ASD_val)
self.metrics.update(
f'MCMC/chain_{idx}/DSC/{structure}',
DSC_val)
no_non_diffeomorphic_voxels_chain = no_non_diffeomorphic_voxels[
idx]
self.metrics.update(
f'MCMC/chain_{idx}/no_non_diffeomorphic_voxels',
no_non_diffeomorphic_voxels_chain)
if no_non_diffeomorphic_voxels_chain > 0.001 * self.no_voxels:
self.logger.info(
f'chain {idx}, sample {sample_no}: '
f'detected {no_non_diffeomorphic_voxels} voxels where '
f'the sampled transformation is not diffeomorphic; exiting..'
)
exit()
residuals = aux['residuals'][idx]
# tensorboard
log_sample(self.writer, idx, im_moving_warped,
v_norm, displacement_norm, log_det_J)
log_hist_res(self.writer,
residuals,
data_loss,
model='MCMC',
chain_no=idx)
# .nii.gz/.vtk
save_sample(self.save_dirs,
self.im_spacing,
sample_no,
im_moving_warped,
displacement,
log_det_J,
'MCMC',
chain_no=idx)
self.logger.info(
'\ncalculating sample standard deviation of the displacement..')
mean, std_dev = calc_posterior_statistics(samples)
log_displacement_mean_and_std_dev(self.writer, mean, std_dev, 'MCMC')
save_displacement_mean_and_std_dev(self.logger, self.save_dirs,
self.im_spacing, mean, std_dev,
moving['mask'], 'MCMC')
"""
speed
"""
no_samples_MCMC_speed_test = 100
start = datetime.now()
for sample_no in range(1, no_samples_MCMC_speed_test + 1):
_, output, _ = self._SGLD_transition(fixed, moving, data_loss,
reg_loss)
seg_moving_warped = self.registration_module(
moving['seg'], output['transformation'])
stop = datetime.now()
MCMC_sampling_speed = self.no_chains * no_samples_MCMC_speed_test / (
stop - start).total_seconds()
self.logger.info(
f'\nMCMC sampling speed: {MCMC_sampling_speed:.2f} samples/sec')
clf = RandomForestClassifier(n_estimators=200,
max_depth=100,
random_state=SEED)
pipeline = Pipeline([('feature_extractor', combined), ('classifier', clf)])
bench = EstimatorPerformance(pipeline)
# training
bench.fit(x_train, y_train)
# predict
predict = bench.predict(x_test)
# calc metrics
utils.calc_metrics(y_test, predict)
# test for competition
utils.test(pipeline)
# importance
# importanceの値が高い順に特徴量をソートする
sorted_features_list = [
f for (idx, f) in sorted(zip(np.argsort(clf.feature_importances_),
features_list),
reverse=True)
]
feature_importance = pd.DataFrame({
"feature":
sorted_features_list,
"importance":
def find_best_algo(
train_data: pd.DataFrame,
target: Union[np.array, pd.Series],
alg_class,
cross_val,
params: List[Mapping],
metric: str,
random_state: int,
early_stopping: int,
) -> Tuple[float, dict, list]:
""""Find best algorithm
Args:
train_data: training data
target: target values
alg_class: algorith class with fit and early_stopping
cross_val: sklearn cross validation class object
params: list of parameters dicts
metric: optimization metric name
random_state: random seed for algorithm
early_stopping: number of early stopping rounds
Return:
best_score (float): best metric score
best_params (dict): best algorithm parameters
best_alg_list (list): list of trained on validation splits algorithms
"""
best_params = None
best_score = None
best_alg_list = None
for param in tqdm(params):
real_values = []
predictions = []
alg_list = []
param["random_state"] = random_state
for train_index, test_index in cross_val.split(train_data):
alg = alg_class(**param)
X_train, X_test = train_data.iloc[train_index], train_data.iloc[
test_index]
y_train, y_test = (
target.iloc[train_index].values,
target.iloc[test_index].values,
)
if param.get("boosting_type", None) != "dart" and param.get(
"boosting_type", None):
alg.fit(
X_train,
y_train,
early_stopping_rounds=early_stopping,
eval_set=[(X_test, y_test)],
verbose=False,
)
else:
alg.fit(X_train, y_train)
alg_list.append(alg)
predictions.append(alg.predict(X_test))
real_values.append(y_test)
metrics_df = calc_metrics(
real_values, predictions,
[mean_absolute_error, mean_squared_error, rmse])
if best_score is None or metrics_df[metric].mean() < best_score:
print("new best score {}+-{:.2f}".format(metrics_df[metric].mean(),
metrics_df[metric].std()))
best_score = metrics_df[metric].mean()
best_params = param
best_alg_list = alg_list
return best_score, best_params, best_alg_list
def main():
args = get_arguments()
if (args.logdir is not None and os.path.isdir(args.logdir)):
logdir = args.logdir
else:
print('Argument --logdir=\'{}\' is not (but should be) '
'a path to valid directory.'.format(args.logdir))
return
with open(args.model_params, 'r') as f:
model_params = json.load(f)
with open(RUNTIME_SWITCHES, 'r') as f:
switch = json.load(f)
receptive_field = WaveNetModel.calculate_receptive_field(
model_params['filter_width'],
model_params['dilations'],
model_params['initial_filter_width'])
# Create coordinator.
coord = tf.train.Coordinator()
# Create data loader.
with tf.name_scope('create_inputs'):
reader = WavMidReader(data_dir=args.data_dir_test,
coord=coord,
audio_sample_rate=model_params['audio_sr'],
receptive_field=receptive_field,
velocity=args.velocity,
sample_size=args.sample_size,
queues_size=(100, 100*BATCH_SIZE))
# Create model.
net = WaveNetModel(
batch_size=BATCH_SIZE,
dilations=model_params['dilations'],
filter_width=model_params['filter_width'],
residual_channels=model_params['residual_channels'],
dilation_channels=model_params['dilation_channels'],
skip_channels=model_params['skip_channels'],
output_channels=model_params['output_channels'],
use_biases=model_params['use_biases'],
initial_filter_width=model_params['initial_filter_width'])
input_data = tf.placeholder(dtype=tf.float32,
shape=(BATCH_SIZE, None, 1))
input_labels = tf.placeholder(dtype=tf.float32,
shape=(BATCH_SIZE, None,
model_params['output_channels']))
_, probs = net.loss(input_data=input_data,
input_labels=input_labels,
pos_weight=1.0,
l2_reg_str=None)
# Set up session
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.trainable_variables())
try:
load(saver, sess, logdir)
except:
print('Something went wrong while restoring checkpoint.')
raise
try:
stats = 0, 0, 0, 0, 0, 0
est = np.empty([model_params['output_channels'], 0])
ref = np.empty([model_params['output_channels'], 0])
sub_fac = int(model_params['audio_sr']/switch['midi_sr'])
for data, labels in reader.single_pass(sess,
args.data_dir_test):
predictions = sess.run(probs, feed_dict={input_data : data})
# Aggregate sums for metrics calculation
stats_chunk = calc_stats(predictions, labels, args.threshold)
stats = tuple([sum(x) for x in zip(stats, stats_chunk)])
est = np.append(est, roll_subsample(predictions.T, sub_fac), axis=1)
ref = np.append(ref, roll_subsample(labels.T, sub_fac, b=True),
axis=1)
metrics = calc_metrics(None, None, None, stats=stats)
write_metrics(metrics, None, None, None, None, None, logdir=logdir)
# Save subsampled data for further arbitrary evaluation
np.save(logdir+'/est.npy', est)
np.save(logdir+'/ref.npy', ref)
# Render evaluation results
figsize=(int(args.plot_scale*est.shape[1]/switch['midi_sr']),
int(args.plot_scale*model_params['output_channels']/12))
if args.media:
write_images(est, ref, switch['midi_sr'],
args.threshold, figsize,
None, None, None, 0, None,
noterange=(21, 109),
legend=args.plot_legend,
logdir=logdir)
write_audio(est, ref, switch['midi_sr'],
model_params['audio_sr'], 0.007,
None, None, None, 0, None, logdir=logdir)
except KeyboardInterrupt:
# Introduce a line break after ^C is displayed so save message
# is on its own line.
print()
finally:
coord.request_stop()
def trainGAN():
req_data = request.get_json()
size_viz = req_data['visualization_size']
nb_batches = req_data['nb_batches']
train_more_Discriminator = req_data['train_more_Discriminator']
train_more_Generator = req_data['train_more_Generator']
unrolling_step = req_data['unrolling_step']
nb_batches = nb_batches
necessary_elements['flip'] = necessary_elements['flip'] if (necessary_elements['flip']) else False
necessary_elements['smooth'] = necessary_elements['smooth'] if (necessary_elements['smooth']) else False
D_Loss_real_min = []
D_Loss_real_mean = []
D_Loss_real_max = []
D_Loss_fake_min = []
D_Loss_fake_mean = []
D_Loss_fake_max = []
fake_generated = []
track_convergence_DS = []
Precision = []
Recall = []
F1_score = []
#Inception_Score = []
generated_bytes = None
real_bytes = None
necessary_elements['discriminator'].to(torch.device(necessary_elements['deviceDiscriminator']))
necessary_elements['generator'].to(torch.device(necessary_elements['deviceGenerator']))
necessary_elements['training'] = True
best_KL = 30
worst_KL = 0
start_time = time.time()
for i in range(nb_batches):
if (unrolling_step != 0):
unrollingIterator = iter(necessary_elements['dataloader'])
for tmpIndex in range(unrolling_step):
unrollingIterItem = next(unrollingIterator)
if(len(unrollingIterItem) == 1):
unrolling_real_batch_element = unrollingIterItem[0]
else: unrolling_real_batch_element,_ = unrollingIterItem
if (necessary_elements['model_discriminator'] != 'DCGAN'):
unrolling_real_batch_element = unrolling_real_batch_element.view(unrolling_real_batch_element.shape[0], necessary_elements['channels']*necessary_elements['img_size']*necessary_elements['img_size'])
else: unrolling_real_batch_element = unrolling_real_batch_element.view(unrolling_real_batch_element.shape[0], necessary_elements['channels'], necessary_elements['img_size'], necessary_elements['img_size'])
unrolling_real_batch.append(unrolling_real_batch_element)
if (necessary_elements['index_batch'] == 0) or (necessary_elements['index_batch'] == len(necessary_elements['dataloader'])-1):
necessary_elements['index_batch'] = 0
necessary_elements['loaderIteraor'] = iter(necessary_elements['dataloader'])
necessary_elements['epoch_number'] = necessary_elements['epoch_number'] + 1
iterItem = next(necessary_elements['loaderIteraor'])
necessary_elements['index_batch'] = necessary_elements['index_batch'] + 1
if(len(iterItem) == 1):
real_batch = iterItem[0]
else: real_batch,_ = iterItem
if (necessary_elements['apply_occasional_flip'] and necessary_elements['index_batch'] % necessary_elements['occasional_flip'] == 0):
necessary_elements['flip'] = True
# train Discriminator:
fake_batch = necessary_elements['generator'](necessary_elements['latentvector']).detach()
real_batch, fake_batch = reshape(necessary_elements['model_discriminator'],real_batch, fake_batch,
necessary_elements['batch_size'], necessary_elements['img_size'], necessary_elements['channels'])
real_batch = real_batch.to(necessary_elements['deviceDiscriminator'])
fake_batch = fake_batch.to(necessary_elements['deviceDiscriminator'])
d_error, d_real, d_fake, gradient_penalty = train_discriminator(necessary_elements['optimizerD'], real_batch,
fake_batch, necessary_elements['discriminator'],
necessary_elements['discriminator_loss_function'],
necessary_elements['deviceDiscriminator'],
necessary_elements['flip'], necessary_elements['smooth'],
necessary_elements['symmetric_labels'],
necessary_elements['apply_gp'], necessary_elements['lambda_gp'],
LVmodel_generator['model_generator'], necessary_elements['clip_d'],
necessary_elements['apply_clip_d'], necessary_elements['apply_divide_d_cost'])
D_Loss.append(d_error.item())
d_real_squeezed = torch.squeeze(d_real)
d_fake_squeezed = torch.squeeze(d_fake)
d_real_squeezed = d_real_squeezed.data.cpu()
d_fake_squeezed = d_fake_squeezed.data.cpu()
D_Loss_real_mean.append(torch.mean(d_real_squeezed).item())
D_Loss_real_min.append(torch.min(d_real_squeezed).item())
D_Loss_real_max.append(torch.max(d_real_squeezed).item())
D_Loss_fake_mean.append(torch.mean(d_fake_squeezed).item())
D_Loss_fake_min.append(torch.min(d_fake_squeezed).item())
D_Loss_fake_max.append(torch.max(d_fake_squeezed).item())
calc_metrics(d_real_squeezed, d_fake_squeezed, Precision, Recall, F1_score)
#when we want to train D more:
if (train_more_Discriminator > 0):
for index in range(train_more_Discriminator):
fake_batch = necessary_elements['generator'](necessary_elements['latentvector']).detach()
_, fake_batch = reshape(necessary_elements['model_discriminator'],real_batch, fake_batch,
necessary_elements['batch_size'], necessary_elements['img_size'], necessary_elements['channels'])
fake_batch = fake_batch.to(necessary_elements['deviceDiscriminator'])
d_error, d_real, d_fake, gradient_penalty = train_discriminator(necessary_elements['optimizerD'], real_batch,
fake_batch, necessary_elements['discriminator'],
necessary_elements['discriminator_loss_function'],
necessary_elements['deviceDiscriminator'],
necessary_elements['flip'], necessary_elements['smooth'],
necessary_elements['symmetric_labels'],
necessary_elements['apply_gp'], necessary_elements['lambda_gp'],
LVmodel_generator['model_generator'], necessary_elements['clip_d'],
necessary_elements['apply_clip_d'], necessary_elements['apply_divide_d_cost'])
D_Loss.append(d_error.item())
d_real_squeezed = torch.squeeze(d_real)
d_fake_squeezed = torch.squeeze(d_fake)
d_real_squeezed = d_real_squeezed.data.cpu()
d_fake_squeezed = d_fake_squeezed.data.cpu()
D_Loss_real_mean.append(torch.mean(d_real_squeezed).item())
D_Loss_real_min.append(torch.min(d_real_squeezed).item())
D_Loss_real_max.append(torch.max(d_real_squeezed).item())
D_Loss_fake_mean.append(torch.mean(d_fake_squeezed).item())
D_Loss_fake_min.append(torch.min(d_fake_squeezed).item())
D_Loss_fake_max.append(torch.max(d_fake_squeezed).item())
#train Generator:
fake_generated = necessary_elements['generator'](necessary_elements['latentvector'])
_, fake_generated = reshape(necessary_elements['model_discriminator'],real_batch, fake_generated,
necessary_elements['batch_size'], necessary_elements['img_size'], necessary_elements['channels'])
g_error = train_generator(necessary_elements['optimizerG'], fake_generated,
necessary_elements['discriminator'], necessary_elements['generator_loss_function'],
necessary_elements['deviceGenerator'], necessary_elements['deviceDiscriminator'],
necessary_elements['flip'], necessary_elements['smooth'],
necessary_elements['symmetric_labels'],
unrolling_step, necessary_elements['optimizerD'],
unrolling_real_batch, necessary_elements['discriminator_loss_function'],
real_batch, necessary_elements['apply_feature_matching'],
necessary_elements['batch_size'])
G_Loss.append(g_error.item())
kl_div_item = (F.kl_div(fake_generated, real_batch)).item()
KL_div.append(abs(kl_div_item))
if (abs(kl_div_item) < best_KL):
best_KL = abs(kl_div_item)
best_generated = fake_generated
if (abs(kl_div_item) > worst_KL):
worst_KL = abs(kl_div_item)
worst_generated = fake_generated
tmpJS = 0.5 * (real_batch + fake_generated)
JS_div.append(abs((0.5*(F.kl_div(real_batch, tmpJS) + F.kl_div(fake_generated, tmpJS))).item()))
#when we want to train G more:
if (train_more_Generator > 0):
for index in range(train_more_Generator):
fake_generated = necessary_elements['generator'](necessary_elements['latentvector'])
_, fake_batch = reshape(necessary_elements['model_discriminator'],real_batch, fake_batch,
necessary_elements['batch_size'], necessary_elements['img_size'], necessary_elements['channels'])
g_error = train_generator(necessary_elements['optimizerG'], fake_generated,
necessary_elements['discriminator'], necessary_elements['generator_loss_function'],
necessary_elements['deviceGenerator'], necessary_elements['deviceDiscriminator'],
necessary_elements['flip'], necessary_elements['smooth'],
necessary_elements['symmetric_labels'],
unrolling_step, necessary_elements['optimizerD'],
unrolling_real_batch, necessary_elements['discriminator_loss_function'],
real_batch, necessary_elements['apply_feature_matching'],
necessary_elements['batch_size'])
G_Loss.append(g_error.item())
KL_div.append(abs((F.kl_div(fake_generated, real_batch)).item()))
tmpJS = 0.5 * (real_batch + fake_generated)
JS_div.append(abs((0.5*(F.kl_div(real_batch, tmpJS) + F.kl_div(fake_generated, tmpJS))).item()))
if (necessary_elements['apply_occasional_flip'] and necessary_elements['index_batch'] % necessary_elements['occasional_flip'] == 0):
necessary_elements['flip'] = False
track_convergence_DS.extend([(F.kl_div(real_batch, fake_generated)).item(),
g_error.item(), d_error.item(),
torch.min(d_real_squeezed).item(), torch.max(d_real_squeezed).item(),
torch.min(d_fake_squeezed).item(), torch.max(d_fake_squeezed).item()])
# loop done:
result_elements['d_error'] = d_error.tolist()
result_elements['d_real'] = d_real.tolist()
result_elements['d_fake'] = d_fake.tolist()
# 2D data:
best_js_sample = 0
fake_sample = worst_generated[0]
real_sample = real_batch[0]
for real_sample_item in real_batch :
tmpJS_sample = 0.5 * (real_sample_item + fake_sample)
js_div_item_sample = (0.5*(F.kl_div(real_sample_item, tmpJS_sample) + F.kl_div(fake_sample, tmpJS_sample))).item()
if (js_div_item_sample > best_js_sample):
best_js_sample = js_div_item_sample
real_sample = real_sample_item
real_sample, fake_sample = rgb_to_gray(real_sample, fake_sample, necessary_elements['img_size'], necessary_elements['channels'])
result_elements['pca_real2D'] = pca_real2D.fit_transform(real_sample.data.cpu())
result_elements['pca_generated2D'] = pca_generated2D.fit_transform(fake_sample.data.cpu())
# 3D data:
result_elements['pca_real3D'] = pca_real3D.fit_transform(real_sample.data.cpu())
result_elements['pca_generated3D'] = pca_generated3D.fit_transform(fake_sample.data.cpu())
result_elements['g_error'] = g_error.tolist()
tmpFakeGenerated = best_generated[:size_viz * size_viz]
tmpWorstGenerated = worst_generated[:size_viz * size_viz]
tmpFakeGenerated = vector_to_image(tmpFakeGenerated, size_viz * size_viz).data.cpu()
tmpWorstGenerated = vector_to_image(tmpWorstGenerated, size_viz * size_viz).data.cpu()
if type(tmpFakeGenerated) == np.ndarray:
tmpFakeGenerated = torch.from_numpy(tmpFakeGenerated)
if type(tmpWorstGenerated) == np.ndarray:
tmpWorstGenerated = torch.from_numpy(tmpWorstGenerated)
gridGenerated = vutils.make_grid(tmpFakeGenerated, nrow=size_viz, normalize=True, scale_each=True)
gridWorstGenerated = vutils.make_grid(tmpWorstGenerated, nrow=size_viz, normalize=True, scale_each=True)
tmpRealImages = real_batch[:size_viz * size_viz]
tmpRealImages = vector_to_image(tmpRealImages, size_viz * size_viz).data.cpu()
if type(tmpRealImages) == np.ndarray:
tmpRealImages = torch.from_numpy(tmpRealImages)
gridReal = vutils.make_grid(tmpRealImages, nrow=size_viz, normalize=True, scale_each=True)
tmpFakeGenerated = np.transpose(gridGenerated.cpu(), (1, 2, 0))
strIO = BytesIO()
imsave(strIO, tmpFakeGenerated, plugin='pil', format_str='png')
strIO.seek(0)
generated_bytes = base64.b64encode(strIO.getvalue())
tmpWorstGenerated = np.transpose(gridWorstGenerated.cpu(), (1, 2, 0))
strIO = BytesIO()
imsave(strIO, tmpWorstGenerated, plugin='pil', format_str='png')
strIO.seek(0)
worst_generated_bytes = base64.b64encode(strIO.getvalue())
tmpRealImages = np.transpose(gridReal.cpu(), (1, 2, 0))
strIO = BytesIO()
imsave(strIO, tmpRealImages, plugin='pil', format_str='png')
strIO.seek(0)
real_bytes = base64.b64encode(strIO.getvalue())
end_time = time.time()
elapsed_time = end_time - start_time
if (result_elements != None):
return jsonify(
d_error= D_Loss[(len(D_Loss) - 40):],
g_error= G_Loss[(len(G_Loss) - 40):],
size_generated_images= fake_generated[:size_viz * size_viz].shape,
d_Loss_real_min= D_Loss_real_min,
d_Loss_real_mean= D_Loss_real_mean,
d_Loss_real_max= D_Loss_real_max,
d_Loss_fake_min= D_Loss_fake_min,
d_Loss_fake_mean= D_Loss_fake_mean,
d_Loss_fake_max= D_Loss_fake_max,
generated_bytes= generated_bytes.decode('ascii'),
worst_generated_bytes= worst_generated_bytes.decode('ascii'),
real_bytes= real_bytes.decode('ascii'),
real_2d= [result_elements['pca_real2D'][:, 0].tolist() , result_elements['pca_real2D'][:, 1].tolist()],
fake_2d= [result_elements['pca_generated2D'][:, 0].tolist() , result_elements['pca_generated2D'][:, 1].tolist() ],
real_3d= [result_elements['pca_real3D'][:, 0].tolist() , result_elements['pca_real3D'][:, 1].tolist(), result_elements['pca_real3D'][:, 2].tolist()],
fake_3d= [result_elements['pca_generated3D'][:, 0].tolist() , result_elements['pca_generated3D'][:, 1].tolist(), result_elements['pca_generated3D'][:, 2].tolist()],
kl_div= KL_div[(len(KL_div) - 40):],
js_div= JS_div[(len(JS_div) - 40):],
precision= Precision,
recall= Recall,
f1_score= F1_score,
training= necessary_elements['training'],
track_convergence_DS= track_convergence_DS,
elapsed_time= elapsed_time,
index_batch= necessary_elements['index_batch'],
epoch_number= necessary_elements['epoch_number'],
status= 200,
mimetype='application/json'
)
else:
return app.response_class(
response = json.dumps(result_elements),
status = 400,
mimetype='application/json'
)
