def main(args):
# Check if the output folder is exist
if not os.path.exists(args.folder):
os.mkdir(args.folder)
# Load model
model = CVAE().cuda() if torch.cuda.is_available() else CVAE()
model.load_state_dict(torch.load(os.path.join(args.folder, 'cvae.pth')))
# Generate
sample = torch.randn(args.num, 20)
label = torch.from_numpy(np.asarray([args.digits] * args.num))
sample = Variable(
sample).cuda() if torch.cuda.is_available() else Variable(sample)
sample = model.decode(sample, label).cpu()
save_image(sample.view(args.num, 1, 28, 28).data,
os.path.join(args.folder, 'generate.png'),
nrow=10)
def train(train_A_dir,
train_B_dir,
model_dir,
model_name,
random_seed,
val_A_dir,
val_B_dir,
output_dir,
tensorboard_dir,
load_path,
gen_eval=True):
np.random.seed(random_seed)
# For now, copy hyperparams used in the CycleGAN
num_epochs = 100000
mini_batch_size = 1 # mini_batch_size = 1 is better
learning_rate = 0.0002
learning_rate_decay = learning_rate / 200000
sampling_rate = 16000
num_mcep = 24
frame_period = 5.0
n_frames = 128
lambda_cycle = 10
lambda_identity = 5
device = 'cuda'
# Use the same pre-processing as the CycleGAN
print("Begin Preprocessing")
wavs_A = load_wavs(wav_dir=train_A_dir, sr=sampling_rate)
wavs_B = load_wavs(wav_dir=train_B_dir, sr=sampling_rate)
print("Finished Loading")
f0s_A, timeaxes_A, sps_A, aps_A, coded_sps_A = world_encode_data(
wavs=wavs_A,
fs=sampling_rate,
frame_period=frame_period,
coded_dim=num_mcep)
f0s_B, timeaxes_B, sps_B, aps_B, coded_sps_B = world_encode_data(
wavs=wavs_B,
fs=sampling_rate,
frame_period=frame_period,
coded_dim=num_mcep)
print("Finished Encoding")
log_f0s_mean_A, log_f0s_std_A = logf0_statistics(f0s_A)
log_f0s_mean_B, log_f0s_std_B = logf0_statistics(f0s_B)
print('Log Pitch A')
print('Mean: %f, Std: %f' % (log_f0s_mean_A, log_f0s_std_A))
print('Log Pitch B')
print('Mean: %f, Std: %f' % (log_f0s_mean_B, log_f0s_std_B))
coded_sps_A_transposed = transpose_in_list(lst=coded_sps_A)
coded_sps_B_transposed = transpose_in_list(lst=coded_sps_B)
coded_sps_A_norm, coded_sps_A_mean, coded_sps_A_std = coded_sps_normalization_fit_transoform(
coded_sps=coded_sps_A_transposed)
print("Input data fixed.")
coded_sps_B_norm, coded_sps_B_mean, coded_sps_B_std = coded_sps_normalization_fit_transoform(
coded_sps=coded_sps_B_transposed)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
np.savez(os.path.join(model_dir, 'logf0s_normalization.npz'),
mean_A=log_f0s_mean_A,
std_A=log_f0s_std_A,
mean_B=log_f0s_mean_B,
std_B=log_f0s_std_B)
np.savez(os.path.join(model_dir, 'mcep_normalization.npz'),
mean_A=coded_sps_A_mean,
std_A=coded_sps_A_std,
mean_B=coded_sps_B_mean,
std_B=coded_sps_B_std)
if val_A_dir is not None:
validation_A_output_dir = os.path.join(output_dir, 'converted_A')
if not os.path.exists(validation_A_output_dir):
os.makedirs(validation_A_output_dir)
if val_B_dir is not None:
validation_B_output_dir = os.path.join(output_dir, 'converted_B')
if not os.path.exists(validation_B_output_dir):
os.makedirs(validation_B_output_dir)
print("End Preprocessing")
if load_path is not None:
model = CVAE(num_mcep, 128, num_mcep, 2)
model.load_state_dict(torch.load(load_path))
model.eval()
if device == 'cuda':
model.cuda()
print("Loaded Model from path %s" % load_path)
if val_A_dir is not None and gen_eval:
print("Generating Evaluation Data")
for file in os.listdir(val_A_dir):
filepath = os.path.join(val_A_dir, file)
print(
"Converting {0} from Class 0 to Class 1".format(filepath))
wav, _ = librosa.load(filepath, sr=sampling_rate, mono=True)
wav = wav_padding(wav=wav,
sr=sampling_rate,
frame_period=frame_period,
multiple=4)
f0, timeaxis, sp, ap = world_decompose(
wav=wav, fs=sampling_rate, frame_period=frame_period)
f0_converted = pitch_conversion(f0=f0,
mean_log_src=log_f0s_mean_A,
std_log_src=log_f0s_std_A,
mean_log_target=log_f0s_mean_B,
std_log_target=log_f0s_std_B)
coded_sp = world_encode_spectral_envelop(sp=sp,
fs=sampling_rate,
dim=num_mcep)
coded_sp_transposed = coded_sp.T
coded_sp_norm = (coded_sp_transposed -
coded_sps_A_mean) / coded_sps_A_std
coded_sp_converted_norm, _, _ = model.convert(
np.array([coded_sp_norm]), 0, 1, device)
coded_sp_converted_norm = coded_sp_converted_norm.cpu().numpy()
coded_sp_converted_norm = np.squeeze(coded_sp_converted_norm)
coded_sp_converted = coded_sp_converted_norm * coded_sps_B_std + coded_sps_B_mean
coded_sp_converted = coded_sp_converted.T
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
decoded_sp_converted = world_decode_spectral_envelop(
coded_sp=coded_sp_converted, fs=sampling_rate)
wav_transformed = world_speech_synthesis(
f0=f0_converted,
decoded_sp=decoded_sp_converted,
ap=ap,
fs=sampling_rate,
frame_period=frame_period)
librosa.output.write_wav(
os.path.join(validation_A_output_dir,
'eval_' + os.path.basename(file)),
wav_transformed, sampling_rate)
exit(0)
print("Begin Training")
model = CVAE(num_mcep, 128, num_mcep, 2)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
writer = SummaryWriter(tensorboard_dir)
if device == 'cuda':
model.cuda()
for epoch in tqdm(range(num_epochs)):
dataset_A, dataset_B = sample_train_data(dataset_A=coded_sps_A_norm,
dataset_B=coded_sps_B_norm,
n_frames=n_frames)
dataset_A = torch.tensor(dataset_A).to(torch.float)
dataset_B = torch.tensor(dataset_B).to(torch.float)
n_samples, input_dim, depth = dataset_A.shape
y_A = F.one_hot(torch.zeros(depth).to(torch.int64),
num_classes=2).to(torch.float).T
y_B = F.one_hot(torch.ones(depth).to(torch.int64),
num_classes=2).to(torch.float).T
(y_A, y_B) = (y_A.reshape((1, 2, depth)), y_B.reshape((1, 2, depth)))
y_A = torch.cat([y_A] * n_samples)
y_B = torch.cat([y_B] * n_samples)
# dataset_A = torch.cat((dataset_A, y_A), axis=1)
# dataset_B = torch.cat((dataset_B, y_B), axis=1)
X = torch.cat((dataset_A, dataset_B)).to(device)
Y = torch.cat((y_A, y_B)).to(device)
# out, z_mu, z_var = model(dataset_A, y_A)
# rec_loss = F.binary_cross_entropy(out, dataset_A, size_average=False)
# kl_diver = -0.5 * torch.sum(1 + z_var - z_mu.pow(2) - z_var.exp())
out, z_mu, z_var = model(X, Y)
rec_loss = F.binary_cross_entropy(out, X, size_average=False)
kl_diver = -0.5 * torch.sum(1 + z_var - z_mu.pow(2) - z_var.exp())
loss = rec_loss + kl_diver
writer.add_scalar('Reconstruction Loss', rec_loss, epoch)
writer.add_scalar('KL-Divergence', kl_diver, epoch)
writer.add_scalar('Total Loss', loss, epoch)
# print("loss = {0} || rec = {1} || kl = {2}".format(loss, rec_loss, kl_diver))
loss.backward()
optimizer.step()
if val_A_dir is not None:
if epoch % 1000 == 0:
print('Generating Validation Data...')
for file in os.listdir(val_A_dir):
filepath = os.path.join(val_A_dir, file)
print("Converting {0} from Class 0 to Class 1".format(
filepath))
wav, _ = librosa.load(filepath,
sr=sampling_rate,
mono=True)
wav = wav_padding(wav=wav,
sr=sampling_rate,
frame_period=frame_period,
multiple=4)
f0, timeaxis, sp, ap = world_decompose(
wav=wav, fs=sampling_rate, frame_period=frame_period)
f0_converted = pitch_conversion(
f0=f0,
mean_log_src=log_f0s_mean_A,
std_log_src=log_f0s_std_A,
mean_log_target=log_f0s_mean_B,
std_log_target=log_f0s_std_B)
coded_sp = world_encode_spectral_envelop(sp=sp,
fs=sampling_rate,
dim=num_mcep)
coded_sp_transposed = coded_sp.T
coded_sp_norm = (coded_sp_transposed -
coded_sps_A_mean) / coded_sps_A_std
coded_sp_converted_norm, _, _ = model.convert(
np.array([coded_sp_norm]), 0, 1, device)
coded_sp_converted_norm = coded_sp_converted_norm.cpu(
).numpy()
coded_sp_converted_norm = np.squeeze(
coded_sp_converted_norm)
coded_sp_converted = coded_sp_converted_norm * coded_sps_B_std + coded_sps_B_mean
coded_sp_converted = coded_sp_converted.T
coded_sp_converted = np.ascontiguousarray(
coded_sp_converted)
decoded_sp_converted = world_decode_spectral_envelop(
coded_sp=coded_sp_converted, fs=sampling_rate)
wav_transformed = world_speech_synthesis(
f0=f0_converted,
decoded_sp=decoded_sp_converted,
ap=ap,
fs=sampling_rate,
frame_period=frame_period)
librosa.output.write_wav(
os.path.join(validation_A_output_dir,
str(epoch) + '_' +
os.path.basename(file)), wav_transformed,
sampling_rate)
break
if epoch % 1000 == 0:
print('Saving Checkpoint')
filepath = os.path.join(model_dir, model_name)
if not os.path.exists(filepath):
os.makedirs(filepath)
torch.save(model.state_dict(),
os.path.join(filepath, '{0}.ckpt'.format(epoch)))
_, c_pairs = readLangs("test_c", "./data/test_c.txt")
total_bleu_score = 0.0
for pair, c_pair in zip(pairs, c_pairs):
print('>', pair[0])
print('=', pair[1])
output_words = evaluate(model, c_pair, pair[0])
output_sentence = ''.join(output_words)
print('<', output_sentence)
print('')
total_bleu_score += bleu_score(pair[1], output_sentence)
_bleu_score = total_bleu_score / len(pairs)
print(f"Bleu Score: {_bleu_score}")
return _bleu_score
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
tense2index = {'sp': 0, 'tp': 1, 'pg': 2, 'p': 3}
latent_size = 32
hidden_size = 256
input_lang = torch.load("./lang_class.pth")
checkpoint = torch.load("./checkpoint/0.7902374299152759_61208.pth")
model = CVAE(28, hidden_size, latent_size, 28).to(device)
model.load_state_dict(checkpoint['state_dict'])
evaluateByTestData(model)
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.Resize((224, 224), Image.LANCZOS),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
pad_idx = vocab.word2idx['<pad>']
sos_idx = vocab.word2idx['<start>']
eos_idx = vocab.word2idx['<end>']
unk_idx = vocab.word2idx['<unk>']
# Build the models
model = CVAE(
vocab_size=len(vocab),
embedding_size=args.embedding_size,
rnn_type=args.rnn_type,
hidden_size=args.hidden_size,
word_dropout=args.word_dropout,
embedding_dropout=args.embedding_dropout,
latent_size=args.latent_size,
max_sequence_length=args.max_sequence_length,
num_layers=args.num_layers,
bidirectional=args.bidirectional,
pad_idx=pad_idx,
sos_idx=sos_idx,
eos_idx=eos_idx,
unk_idx=unk_idx
)
if not os.path.exists(args.load_checkpoint):
raise FileNotFoundError(args.load_checkpoint)
model.load_state_dict(torch.load(args.load_checkpoint))
print("Model loaded from {}".format(args.load_checkpoint))
model.to(device)
model.eval()
# Build data loader
train_data_loader, valid_data_loader = get_loader(args.train_image_dir, args.val_image_dir,
args.train_caption_path, args.val_caption_path, vocab,
args.batch_size,
shuffle=True, num_workers=args.num_workers)
f1 = open('{}/results/generated_captions.txt'.format(dataset_root_dir), 'w')
f2 = open('{}/results/ground_truth_captions.txt'.format(dataset_root_dir), 'w')
for i, (images, captions, lengths) in enumerate(valid_data_loader):
images = images.to(device)
sampled_ids, z = model.inference(n=args.batch_size, c=images)
sampled_ids_batches = sampled_ids.cpu().numpy() # (batch_size, max_seq_length)
captions = captions.cpu().numpy()
# Convert word_ids to words
for j, sampled_ids in enumerate(sampled_ids_batches):
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
generated_sentence = ' '.join(sampled_caption)
generated_sentence = generated_sentence.rstrip()
generated_sentence = generated_sentence.replace("\n", "")
generated_sentence = "{0}\n".format(generated_sentence)
if j == 0:
print("RE: {}".format(generated_sentence))
f1.write(generated_sentence)
for g, ground_truth_ids in enumerate(captions):
ground_truth_caption = []
for word_id in ground_truth_ids:
word = vocab.idx2word[word_id]
ground_truth_caption.append(word)
if word == '<end>':
break
ground_truth_sentence = ' '.join(ground_truth_caption)
ground_truth_sentence = ground_truth_sentence.rstrip()
ground_truth_sentence = ground_truth_sentence.replace("\n", "")
ground_truth_sentence = "{0}\n".format(ground_truth_sentence)
if g == 0:
print("GT: {}".format(ground_truth_sentence))
f2.write(ground_truth_sentence)
if i % 10 == 0:
print("This is the {0}th batch".format(i))
f1.close()
f2.close()
import pandas as pd
from sklearn.metrics import classification_report
from model import CVAE
import numpy as np
def Loss_function(x_hat, x, mu, logsimga):
reconstraction_loss = F.binary_cross_entropy(x_hat, x, size_average=False)
KL_div = -0.5 * th.sum(1 + logsimga - mu.pow(2) - logsimga.exp())
return reconstraction_loss + KL_div
model = CVAE()
param = th.load('save_model/vae_adadelta300.pth', map_location=lambda x, y: x)
model.load_state_dict(param)
test = pd.read_csv("dataset/VAE_Test+.csv")
testx, testy = np.array(test[test.columns[test.columns != "class"]]), np.array(
pd.get_dummies(test["class"]))
z_dim = 25
n, m = testx.shape[1], testy.shape[1]
test_label = th.eye(m)
attack_name = ["normal", "Dos", "Probe", "R2L", "U2R"]
pred = []
for x in testx:
each_loss = []
x = th.Tensor(x.reshape(1, n))
for label in test_label:
label = th.Tensor(label.reshape(1, m))
x_hat, mu, sigma = model(x, label)
temperature=args.temperature)
if args.load_model is not None:
state_dict = torch.load(args.load_model)
print(state_dict['embedding.weight'].size(),
model.embedding.weight.size())
if state_dict['embedding.weight'].size(
0) != model.embedding.weight.size(0): # vocab changed
state_dict['embedding.weight'] = vocab.vectors
state_dict['outputs2vocab.weight'] = torch.randn(
len(i2w), args.hidden_size * model.hidden_factor)
state_dict['outputs2vocab.bias'] = torch.randn(len(i2w))
print(state_dict['embedding.weight'].size(),
model.embedding.weight.size())
model.load_state_dict(state_dict)
else:
model.embedding.weight.data.copy_(vocab.vectors)
model = to_device(model)
print(model)
train(model, datasets, args)
if args.save_model is not None:
torch.save(model.state_dict(), args.save_model)
if args.n_generated > 0:
model.eval()
samples, z, y_onehot = model.inference(n=args.n_generated)