def run_lower_layers(self):
output = model.encoder(self.data)
for layer in range(self.decomposed_layer_number):
output, self.hidden[layer] = getattr(
self.model,
self.model.rnn_module_name(layer))(output, self.hidden[layer])
return output
def get_clusters(cluster_set):
nDescriptors = 50000
nPerImage = 100
nIm = ceil(nDescriptors / nPerImage)
sampler = SubsetRandomSampler(
np.random.choice(len(cluster_set), nIm, replace=False))
data_loader = DataLoader(dataset=cluster_set,
num_workers=opt.threads,
batch_size=opt.cacheBatchSize,
shuffle=False,
pin_memory=cuda,
sampler=sampler)
if not exists(join(opt.dataPath, 'centroids')):
makedirs(join(opt.dataPath, 'centroids'))
initcache = join(
opt.dataPath, 'centroids', opt.arch + '_' + cluster_set.dataset + '_' +
str(opt.num_clusters) + '_desc_cen.hdf5')
with h5py.File(initcache, mode='w') as h5:
with torch.no_grad():
model.eval()
print('====> Extracting Descriptors')
dbFeat = h5.create_dataset("descriptors",
[nDescriptors, encoder_dim],
dtype=np.float32)
for iteration, (input, indices) in enumerate(data_loader, 1):
input = input.to(device)
image_descriptors = model.encoder(input).view(
input.size(0), encoder_dim, -1).permute(0, 2, 1)
batchix = (iteration - 1) * opt.cacheBatchSize * nPerImage
for ix in range(image_descriptors.size(0)):
# sample different location for each image in batch
sample = np.random.choice(image_descriptors.size(1),
nPerImage,
replace=False)
startix = batchix + ix * nPerImage
dbFeat[startix:startix + nPerImage, :] = image_descriptors[
ix, sample, :].detach().cpu().numpy()
if iteration % 50 == 0 or len(data_loader) <= 10:
print("==> Batch ({}/{})".format(
iteration, ceil(nIm / opt.cacheBatchSize)),
flush=True)
del input, image_descriptors
print('====> Clustering..')
niter = 100
kmeans = faiss.Kmeans(encoder_dim,
opt.num_clusters,
niter,
verbose=False)
kmeans.train(dbFeat[...])
print('====> Storing centroids', kmeans.centroids.shape)
h5.create_dataset('centroids', data=kmeans.centroids)
print('====> Done!')
def train():
# few things that we have define
batch_size = 32
train = True
transform_train = transforms.Compose([
transforms.Resize(256), # smaller edge of image resized to 256
transforms.RandomCrop(224), # get 224x224 crop from random location
transforms.ToTensor(), # convert the PIL Image to a tensor
transforms.Normalize(
(0.485, 0.456, 0.406), # normalize image for pre-trained model
(0.229, 0.224, 0.225))
])
iteration = 3
vocabulary_threshold = 5
embed_size = 512
hidden_size = 512
hidden_layer = 1
model_save = "model_storage/"
# calling the dataloader
train_dataLoader = get_data_loader(vocabulary_threshold, train, batch_size,
transform_train)
enc = encoder(embed_size, batch_size)
dec = decoder(len(train_dataLoader.dataset.vocab.word_to_index),
embed_size, hidden_layer, hidden_size)
params = list(enc.dense.parameters()) + list(dec.parameters())
criteria = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params,
lr=0.001,
betas=(0.9, 0.999),
eps=1e-08)
steps_per_epoch = int(
np.math.ceil(len(train_dataLoader.dataset.caption_len) / batch_size))
for epoch in range(iteration):
for step in range(steps_per_epoch):
index = train_dataLoader.dataset.trainIndices(batch_size)
sampler = torch.utils.data.SubsetRandomSampler(index)
train_dataLoader.batch_sampler.sampler = sampler
img, caption = next(iter(train_dataLoader))
enc.zero_grad()
dec.zero_grad()
features = enc(img)
prediction = dec(features, caption)
loss = criteria(
prediction.view(caption.size(0) * caption.size(1), -1),
caption.view(-1))
loss.backward()
optimizer.step()
stats = "[%d/%d] LOSS: %.4f, PERPLEXITY: %5.4f " % (
step, iteration, loss.item(), np.exp(loss.item()))
print("\r " + stats, end="")
sys.stdout.flush()
if step % 1000 == 0 and step != 0:
# here we save the weights
torch.save({"model_state": enc.state_dict()},
model_save + "encoder_" + str(step) + ".pth")
torch.save({"model_state": dec.state_dict()},
model_save + "decoder_" + str(step) + ".pth")
print("\r" + stats)
def test():
embed_size = 512
hidden_size = 512
weights = "model_storage/"
weight_list = os.listdir(weights)
selectedWeight = None
index_to_word = readVocab()
maxVal = 0
transform_train = transforms.Compose([
transforms.Resize(256), # smaller edge of image resized to 256
transforms.RandomCrop(224), # get 224x224 crop from random location
transforms.ToTensor(), # convert the PIL Image to a tensor
transforms.Normalize(
(0.485, 0.456, 0.406), # normalize image for pre-trained model
(0.229, 0.224, 0.225))
])
for weight in weight_list:
if "encoder" in weight:
val = int(weight.split(".")[0].split("_")[1])
if val > maxVal:
selectedWeight = weight
maxVal = val
encoder_weight = selectedWeight
decoder_weight = selectedWeight.replace("encoder", "decoder")
enc_weight = torch.load(weights + encoder_weight)
dec_weight = torch.load(weights + decoder_weight)
enc = encoder(embed_size, batch_size=1)
enc.eval()
enc.load_state_dict(enc_weight["model_state"])
dec = decoder(len(index_to_word), embed_size, 1, hidden_size)
dec.eval()
dec.load_state_dict(dec_weight["model_state"])
test_loader = get_data_loader(5, False, 1, transform_train)
img_test, original_img = next(iter(test_loader))
features = enc(img_test)
output = dec.sample(features.unsqueeze(1), 20)
sentence = ""
for val in output:
if val != 0 and val != 1 and val != 2:
sentence += index_to_word[val] + " "
plt.imshow(np.uint8(original_img.squeeze(0).numpy()))
plt.text(100,
400,
sentence,
style='italic',
bbox={
'facecolor': 'red',
'alpha': 0.5,
'pad': 10
})
plt.show()
def evaluate(split, verbose=False, n_batches=None):
# Recall model is a class that inherits nn.Module that we learned in the class.
# This puts the model in eval mode as opposed to train mode, so it knows which one to use.
model.encoder.eval()
model.decoder.eval()
# Initialize cumulative loss and the number of correctly predicted examples.
loss = 0
correct = 0
n_examples = 0
# Load the correct dataset between validation.
if split == 'val':
loader = val_loader
# For each batch in the loaded dataset,
with torch.no_grad():
for batch_i, batch in enumerate(loader):
data, caption, lengths = batch[0], batch[1], batch[2]
targets = pack_padded_sequence(caption, lengths,
batch_first=True)[0]
# Load the current training example in the CUDA core if available.
if args.cuda:
data, caption = data.cuda(), caption.cuda()
# Read images and their target labels in the current batch.
data, caption = Variable(data), Variable(caption)
# Measure the output results given the data.
features = model.encoder(data)
output = model.decoder(features, caption, lengths)
# Accumulate the loss by comparing the predicted output and the true targets ( both are in pack padded sequence).
loss += criterion(output, targets).data
# Skip the rest of evaluation if the number of batches exceed the n_batches.
if n_batches and (batch_i >= n_batches):
break
# Compute the average loss per example.
loss /= (batch_i + 1)
# If verbose is True, then print out the average loss and accuracy.
if verbose:
print('\n{} set: Average loss: {:.4f}'.format(split, loss))
return loss
def run(self):
batch_size = self.config['batch_size']
learning_rate = self.config['learning_rate']
# Create Model
self.encoder = model.encoder().cuda()
self.decoder = model.decoder().cuda()
self.logger.debug('Encoder Architecture')
summary(self.encoder, (3, 224, 224), batch_size=batch_size)
self.logger.debug('Decoder Architecture')
summary(self.decoder, (512, 14, 14), batch_size=batch_size)
model_params = []
model_params += self.encoder.parameters()
model_params += self.decoder.parameters()
self.optm = torch.optim.SGD(model_params,
lr=learning_rate,
momentum=self.config['momentum'],
weight_decay=self.config['weight_decay'])
# Restore Model
if not self.args.restart:
self.load_checkpoint()
# Setup Global Train Index
self.gidx = self.epoch * len(self.dataset_train)
# Initial Validation
self.valid = DataObject()
self.run_valid()
total_epochs = self.config['epochs']
for _ in range(self.epoch, total_epochs):
utils.adjust_learning_rate(learning_rate, self.optm, self.epoch)
self.train = DataObject()
self.run_train()
self.valid = DataObject()
self.run_valid()
self.epoch += 1
def calculate_sims(word_pair_sim):
model.eval()
model_sims, human_sims = [], []
for (word_a, word_b, sim) in word_pair_sim:
if corpus.dictionary.has_word(word_a) and corpus.dictionary.has_word(
word_b):
human_sims.append(sim)
idx = []
for word in [word_a, word_b]:
idx += [corpus.dictionary.get_idx(word)]
idx = torch.tensor(idx).type(torch.int64)
idx = idx.unsqueeze(0).to(device) # [1, 2]
emb = model.encoder(idx) # [1, 2, hidden]
model_sim = F.cosine_similarity(emb[:, 0],
emb[:,
1]).cpu().detach().numpy()[0]
model_sims.append(model_sim)
return human_sims, model_sims
def main(batch_size, train_df, trainLoader, embedding_dim, hidden_size, hidden_layer, index_to_word):
vocab_size = len(index_to_word)+1
enc = encoder(embedding_dim, batch_size)
dec = decoder(vocab_size, embedding_dim, hidden_layer, hidden_size)
iteration = 10
#loss
param = list(enc.dense.parameters()) + list(dec.parameters())
criteria = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(param, lr=0.001)
total_steps = int(np.ceil(train_size/ batch_size))
caption_len = captionLength(train_df)
for epoch in range(iteration):
total_loss = 0.0
for step in range(total_steps):
train_indices, _ = randomSelect(caption_len, batch_size)
new_sampler = torch.utils.data.SubsetRandomSampler(train_indices)
trainLoader.batch_sampler.sampler = new_sampler
data = next(iter(trainLoader))
original_img, caption = data
enc.zero_grad()
dec.zero_grad()
features = enc(original_img)
prediction = dec(features.long(), caption)
#loss
loss = criteria(prediction.view(caption.size(0)*caption.size(1),-1), caption.view(-1))
loss.backward()
optimizer.step()
stats = "[%d/%d] Loss: %.4f, Perplexity: %5.4f "%(step, iteration, loss.item(), np.exp(loss.item()))
print("\r" +stats, end="")
sys.stdout.flush()
total_loss += loss.item()
if step % 100 ==0 and step != 0:
torch.save({
'epoch': epoch,
'model_state_dict': enc.state_dict(),
'loss': total_loss/100,
}, "loss_folder/encoder_"+str(epoch)+".pth")
torch.save({
'model_state_dict':dec.state_dict()
},"loss_folder/decoder_"+str(epoch)+".pth")
total_loss = 0.0
print("\r" + stats)
def __init__(self,
n_dim=2,
batch_size=100,
epochs=10,
log_freq=100,
results_path='./results',
make_gif=False):
self.n_dim = n_dim
self.batch_size = batch_size
self.epochs = epochs
self.log_freq = log_freq
self.results_path = results_path
self.results_img_path = results_path + "/imges"
self.make_gif = make_gif
if not os.path.exists(self.results_img_path):
os.makedirs(self.results_img_path)
if self.make_gif and not os.path.exists(self.results_path + "/gif"):
os.makedirs(self.results_path + "/gif")
# data load
self.load_data()
self.dataset_train = tf.data.Dataset.from_tensor_slices(
(self.x_train, self.y_train))
self.dtrain_shuffle = self.dataset_train.shuffle(
self.x_train.shape[0]).batch(self.batch_size)
self.dataset_test = tf.data.Dataset.from_tensor_slices(
(self.x_test, self.y_test))
self.dtest_shuffle = self.dataset_test.shuffle(
self.x_test.shape[0]).batch(1000)
# Models
self.encoder = encoder(n_dim=self.n_dim)
self.decoder = decoder()
self.discriminator = discriminator()
# optimizer
self.ae_opt = tf.keras.optimizers.Adam(0.0001)
self.gen_opt = tf.keras.optimizers.Adam(0.0001, beta_1=0, beta_2=0.9)
self.disc_opt = tf.keras.optimizers.Adam(0.0001, beta_1=0, beta_2=0.9)
self.loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=True)
def test( batch_size, df, testLoader, index_to_word):
enc = encoder(512, batch_size)
enc.eval()
dec = decoder(len(index_to_word)+1, 512, 1, 512)
dec.eval()
#load the model
enc_weight = torch.load("loss_folder/encoder_2.pth")
dec_weight = torch.load("loss_folder/decoder_2.pth")
enc.load_state_dict(enc_weight["model_state_dict"])
dec.load_state_dict(dec_weight["model_state_dict"])
img, caption = next(iter(testLoader))
print(img.shape)
caption = caption[0]
features = enc(img).unsqueeze(1)
output = dec.sample(features.float(), 27)
sent = ""
for word in output:
if index_to_word.get(word) !="START" and index_to_word.get(word)!="END" and word !=0:
sent += index_to_word[word]+" "
print(sent)
plt.imshow(img[0].permute(1,2,0).detach().numpy())
plt.show()
help='model to test')
argparser.add_argument('--draw',
action='store_true',
help='whether draw output')
args = argparser.parse_args()
model_path = os.path.join('model', args.model + '.ckpt')
x = tf.placeholder(tf.float32, [None, 28 * 28])
global_step = tf.Variable(0, name='global_step', trainable=False)
mnist = read_data_sets('tmp/MNIST_data')
with tf.Session() as sess:
c, _ = model.encoder(x)
x_, _ = model.decoder(c)
loss = model.loss(x, x_)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, model_path)
print('"%s" loaded' % (model_path))
eval_x_, eval_loss, step = sess.run([x_, loss, global_step],
feed_dict={x: mnist.test.images})
print('loss: %g' % (eval_loss))
if args.draw:
dirpath = os.path.join('tmp', args.model, str(step))
def train(z_dim=None, model_name=None):
"""
Used to train the autoencoder by passing in the necessary inputs.
:param train_model: True -> Train the model, False -> Load the latest trained model and show the image grid.
:return: does not return anything
"""
X_train, y_train = datasets.create_datasets(retrain=0,
task="aae_wgan_" + str(z_dim),
num_aug=0)
batch_size = BATCH_SIZE
input_dim = X_train.shape[-1]
with tf.device("/gpu:0"):
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
x_input = tf.placeholder(dtype=tf.float32,
shape=[batch_size, input_dim, input_dim, 1],
name='Input')
x_target = tf.placeholder(dtype=tf.float32,
shape=[batch_size, input_dim, input_dim, 1],
name='Target')
real_distribution = tf.placeholder(dtype=tf.float32,
shape=[batch_size, z_dim],
name='Real_distribution')
decoder_input = tf.placeholder(dtype=tf.float32,
shape=[1, z_dim],
name='Decoder_input')
encoder_output = encoder(x_input, reuse=False, is_train=True)
encoder_output_test = encoder(x_input, reuse=True, is_train=False)
d_fake, d_fake_logits = discriminator(encoder_output, reuse=False)
d_real, d_real_logits = discriminator(real_distribution, reuse=True)
d_fake_test, d_fake_logits_test = discriminator(encoder_output,
reuse=True)
d_real_test, d_real_logits_test = discriminator(real_distribution,
reuse=True)
decoder_output, std = decoder(encoder_output,
reuse=False,
is_train=True)
encoder_output_z = encoder(decoder_output, reuse=True, is_train=False)
decoder_output_test, std_ = decoder(encoder_output,
reuse=True,
is_train=False)
encoder_output_z_test = encoder(decoder_output_test,
reuse=True,
is_train=False)
#decoder_image = decoder(decoder_input, reuse=True, is_train=False)
# Autoencoder loss
# summed = tf.reduce_mean(tf.square(decoder_output-x_target),[1,2,3])
summed = tf.reduce_sum(tf.square(decoder_output - x_target), [1, 2, 3])
# sqrt_summed = summed
sqrt_summed = tf.sqrt(summed + 1e-8)
autoencoder_loss = tf.reduce_mean(sqrt_summed)
summed_test = tf.reduce_sum(tf.square(decoder_output_test - x_target),
[1, 2, 3])
# sqrt_summed_test = summed_test
sqrt_summed_test = tf.sqrt(summed_test + 1e-8)
autoencoder_loss_test = tf.reduce_mean(sqrt_summed_test)
# l2 loss of z
enc = tf.reduce_sum(tf.square(encoder_output - encoder_output_z), [1])
encoder_l2loss = tf.reduce_mean(enc)
enc_test = tf.reduce_sum(
tf.square(encoder_output_test - encoder_output_z_test), [1])
encoder_l2loss_test = tf.reduce_mean(enc_test)
dc_loss = tf.reduce_mean(d_real_logits - d_fake_logits)
dc_loss_test = tf.reduce_mean(d_real_logits_test - d_fake_logits_test)
with tf.name_scope("Gradient_penalty"):
eta = tf.placeholder(tf.float32, shape=[batch_size, 1], name="Eta")
interp = eta * real_distribution + (1 - eta) * encoder_output
_, c_interp = discriminator(interp, reuse=True)
# taking the zeroth and only element because tf.gradients returns a list
c_grads = tf.gradients(c_interp, interp)[0]
# L2 norm, reshaping to [batch_size]
slopes = tf.sqrt(tf.reduce_sum(tf.square(c_grads), axis=[1]))
tf.summary.histogram("Critic gradient L2 norm", slopes)
grad_penalty = tf.reduce_mean((slopes - 1)**2)
lambd = 10.0
dc_loss += lambd * grad_penalty
# Generator loss
# generator_loss = tf.reduce_mean(
# tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.ones_like(d_fake), logits=d_fake_logits))
generator_loss = tf.reduce_mean(d_fake_logits)
generator_loss_test = tf.reduce_mean(d_fake_logits_test)
all_variables = tf.trainable_variables()
dc_var = tl.layers.get_variables_with_name('Discriminator', True, True)
en_var = tl.layers.get_variables_with_name('Encoder', True, True)
#print en_var
# dc_var = [var for var in all_variables if 'dc' in var.name]
# en_var = [var for var in all_variables if 'encoder' in var.name]
var_grad_autoencoder = tf.gradients(autoencoder_loss, all_variables)[0]
var_grad_discriminator = tf.gradients(dc_loss, dc_var)[0]
var_grad_generator = tf.gradients(generator_loss, en_var)[0]
# Optimizers
with tf.device("/gpu:0"):
autoencoderl2_optimizer = tf.train.AdamOptimizer(
learning_rate=LR, beta1=0.5,
beta2=0.9).minimize(autoencoder_loss + 0.5 * encoder_l2loss)
autoencoder_optimizer = tf.train.AdamOptimizer(
learning_rate=LR, beta1=0.5,
beta2=0.9).minimize(autoencoder_loss)
discriminator_optimizer = tf.train.AdamOptimizer(
learning_rate=LR, beta1=0.5,
beta2=0.9).minimize(dc_loss, var_list=dc_var)
generator_optimizer = tf.train.AdamOptimizer(learning_rate=LR,
beta1=0.5,
beta2=0.9).minimize(
generator_loss,
var_list=en_var)
tl.layers.initialize_global_variables(sess)
# Reshape immages to display them
input_images = tf.reshape(x_input, [-1, input_dim, input_dim, 1])
generated_images = tf.reshape(decoder_output,
[-1, input_dim, input_dim, 1])
# generated_images = tf.reshape(decoder_output, [-1, 28, 28, 1])
tensorboard_path, saved_model_path, log_path, folder_name = form_results(
)
# bp()
writer = tf.summary.FileWriter(logdir=tensorboard_path,
graph=sess.graph)
# Tensorboard visualization
tf.summary.scalar(name='Autoencoder Loss', tensor=autoencoder_loss)
tf.summary.scalar(name='Autoencoder Test Loss',
tensor=autoencoder_loss_test)
tf.summary.scalar(name='Discriminator Loss', tensor=dc_loss)
tf.summary.scalar(name='Generator Loss', tensor=generator_loss)
tf.summary.scalar(name='Autoencoder z Loss', tensor=encoder_l2loss)
tf.summary.histogram(name='Encoder Distribution',
values=encoder_output)
tf.summary.histogram(name='Real Distribution',
values=real_distribution)
tf.summary.histogram(name='Gradient AE', values=var_grad_autoencoder)
tf.summary.histogram(name='Gradient D', values=var_grad_discriminator)
tf.summary.histogram(name='Gradient G', values=var_grad_generator)
tf.summary.image(name='Input Images',
tensor=input_images,
max_outputs=10)
tf.summary.image(name='Generated Images',
tensor=generated_images,
max_outputs=10)
summary_op = tf.summary.merge_all()
saver = tf.train.Saver()
# Saving the model
step = 0
# with tf.Session() as sess:
with open(log_path + '/log.txt', 'a') as log:
log.write("input_dim: {}\n".format(input_dim))
log.write("z_dim: {}\n".format(z_dim))
log.write("batch_size: {}\n".format(batch_size))
log.write("\n")
for i in range(EPOCHS):
b = 0
for batch in tl.iterate.minibatches(inputs=X_train,
targets=np.zeros(X_train.shape),
batch_size=batch_size,
shuffle=True):
z_real_dist = np.random.normal(0, 1, (batch_size, z_dim)) * 1.
z_real_dist = z_real_dist.astype("float32")
batch_x, _ = batch
batch_x = batch_x[:, :, :, np.newaxis]
#lambda_x = np.max(lambda_grow_max / np.float(i), lambda_grow_max)
sess.run(autoencoderl2_optimizer,
feed_dict={
x_input: batch_x,
x_target: batch_x
})
if i < 20:
# sess.run(autoencoder_optimizer, feed_dict={x_input: batch_x, x_target: batch_x})
for t in range(10):
for _ in range(20):
eta1 = np.random.rand(
batch_size,
1) # sampling from uniform distribution
eta1 = eta1.astype("float32")
sess.run(discriminator_optimizer,
feed_dict={
x_input: batch_x,
x_target: batch_x,
real_distribution: z_real_dist,
eta: eta1
})
else:
# sess.run(autoencoderl2_optimizer, feed_dict={x_input: batch_x, x_target: batch_x})
for _ in range(20):
eta1 = np.random.rand(
batch_size, 1) # sampling from uniform distribution
eta1 = eta1.astype("float32")
sess.run(discriminator_optimizer,
feed_dict={
x_input: batch_x,
x_target: batch_x,
real_distribution: z_real_dist,
eta: eta1
})
sess.run(generator_optimizer,
feed_dict={
x_input: batch_x,
x_target: batch_x
})
if b % 50 == 0:
a_loss, e_loss, d_loss, g_loss, a_grad, d_grad, g_grad, en_output, d_real_logits_, d_fake_logits_, de_output, summary = sess.run(
[
autoencoder_loss, encoder_l2loss, dc_loss,
generator_loss, var_grad_autoencoder,
var_grad_discriminator, var_grad_generator,
encoder_output, d_real_logits, d_fake_logits,
decoder_output, summary_op
],
feed_dict={
x_input: batch_x,
x_target: batch_x,
real_distribution: z_real_dist,
eta: eta1
})
print(model_name)
saver.save(sess, save_path=saved_model_path, global_step=step)
writer.add_summary(summary, global_step=step)
print("Epoch: {}, iteration: {}".format(i, b))
print("Autoencoder Loss: {}".format(a_loss))
print("Autoencoder enc Loss: {}".format(e_loss))
print("Discriminator Loss: {}".format(d_loss))
print("Generator Loss: {}".format(g_loss))
with open(log_path + '/log.txt', 'a') as log:
log.write("Epoch: {}, iteration: {}\n".format(i, b))
log.write("Autoencoder Loss: {}\n".format(a_loss))
log.write("Autoencoder enc Loss: {}\n".format(e_loss))
log.write("Discriminator Loss: {}\n".format(d_loss))
log.write("Generator Loss: {}\n".format(g_loss))
b += 1
step += 1
b = 0
for batch in tl.iterate.minibatches(inputs=y_train,
targets=np.zeros(y_train.shape),
batch_size=batch_size,
shuffle=True):
z_real_dist = np.random.normal(0, 1, (batch_size, z_dim)) * 1.
z_real_dist = z_real_dist.astype("float32")
batch_x, _ = batch
batch_x = batch_x[:, :, :, np.newaxis]
eta1 = np.random.rand(batch_size, 1)
if b % 20 == 0:
a_loss, e_loss, d_loss, g_loss = sess.run(
[
autoencoder_loss_test, encoder_l2loss_test,
dc_loss_test, generator_loss_test
],
feed_dict={
x_input: batch_x,
x_target: batch_x,
real_distribution: z_real_dist,
eta: eta1
})
print("v_Epoch: {}, iteration: {}".format(i, b))
print("v_Autoencoder Loss: {}".format(a_loss))
print("v_Autoencoder enc Loss: {}".format(e_loss))
print("v_Discriminator Loss: {}".format(d_loss))
print("v_Generator Loss: {}".format(g_loss))
with open(log_path + '/log.txt', 'a') as log:
log.write("v_Epoch: {}, iteration: {}\n".format(i, b))
log.write("v_Autoencoder Loss: {}\n".format(a_loss))
log.write("v_Autoencoder enc Loss: {}\n".format(e_loss))
log.write("v_Discriminator Loss: {}\n".format(d_loss))
log.write("v_Generator Loss: {}\n".format(g_loss))
print("test shape", test.shape)
test = np.reshape(test, (32, 32, 32)).astype(np.float32)
for_plot.plot_output(test, OUTPUT_SIZE, 'inputCheck')
# ----------------- calculate the number of batches per epoch --------------------
batch_per_ep = input_file.shape[
0] // batch_size # batch per epoch will be 40 [input total= 400 / 10 ]
ae_inputs = tf.placeholder(tf.float32, (None, 32, 32, 32, 1),
name="encoder_input") # input to the network
#dicForShape = tf.placeholder(tf.string, shape=None, name="volume_name")
# ---------for variational auto encoder(this has to be commented when simple auto encoder model is used) --------------
#z_mean, z_std, l_space = md.encoder(ae_inputs)
# ---------for simple auto encoder(this has to be commented when variational model is used) --------------
l_space = md.encoder(ae_inputs, dim_of_z)
# --------- Output from decoder ---------------------
ae_outputs = md.decoder(l_space)
# ----------------- calculate the loss and optimize variational auto encoder network ------------------------
#generation_loss = -tf.reduce_sum(ae_inputs * tf.log(1e-8 + ae_outputs) + (1-ae_inputs) * tf.log(1e-8 + 1 - ae_outputs), 1)
#latent_loss = 0.5 * tf.reduce_sum(tf.square(z_mean) + tf.square(z_std) - tf.log(tf.square(z_std)) - 1,1)
# Voxel-Wise Reconstruction Loss
# Note that the output values are clipped to prevent the BCE from evaluating log(0).
'''ae_outputs = tf.clip_by_value(ae_outputs, 1e-8, 1 - 1e-8)
bce_loss = tf.reduce_sum(weighted_binary_crossentropy(ae_outputs, ae_inputs), [1,2])
bce_loss = tf.reduce_mean(bce_loss)
with open(args.save, 'rb') as f:
# with open("models/candidate.pt", 'rb') as f:
model, model_r, model_mlp, _, _ = torch.load(f)
if args.cuda:
model.cuda()
model_r.cuda()
model_mlp.cuda()
model.eval()
model_r.eval()
seq_len = s1.size(0)
input_emb = model.encoder(s1)
attention, seq_len, reg_len, heights = model_r(input_emb, seq_len)
print(attention.squeeze())
print(heights.squeeze())
# model.eval()
# output2, _, _, A_outputs = model(s2)
# print(output1)
# print(output2)
# A_outputs = [A.cpu().detach().numpy()[0] for A in A_outputs]
# A_outputs.reverse()
# A_last_word = []
def main(src, tar):
clf = encoder().to(device)
domain_clf = domain_classifier().to(device)
optimizer = optim.Adam(list(clf.parameters()) +
list(domain_clf.parameters()),
lr=1e-4)
### dataloader ###
if src == 'mnist':
src_train_set = dset.MNIST('./dataset/mnist',
train=True,
download=True,
transform=gray2rgb_transform)
elif src == 'mnistm':
src_train_set = DATASET('./dataset/mnistm/train',
'./dataset/mnistm/train.csv',
transforms=rgb_transform)
elif src == 'svhn':
src_train_set = dset.SVHN(root='./dataset/svhn/',
download=download,
transform=rgb_transform)
elif src == 'usps':
src_train_set = DATASET('./dataset/usps/train',
'./dataset/usps/train.csv',
transforms=gray2rgb_transform)
if tar == 'svhn':
tar_train_set = dset.SVHN(root='./dataset/svhn/',
download=download,
transform=rgb_transform)
elif tar == 'mnist':
tar_train_set = dset.MNIST('./dataset/mnist',
train=True,
download=True,
transform=gray2rgb_transform)
elif tar == 'mnistm':
tar_train_set = DATASET('./dataset/mnistm/train',
'./dataset/mnistm/train.csv',
transform=rgb_transform)
elif tar == 'usps':
tar_train_set = DATASET('./dataset/usps/train',
'./dataset/usps/train.csv',
transform=rgb_transform)
src_train_loader = torch.utils.data.DataLoader(dataset=src_train_set,
batch_size=BATCH_SIZE,
shuffle=True,
pin_memory=True)
tar_train_loader = torch.utils.data.DataLoader(dataset=tar_train_set,
batch_size=BATCH_SIZE,
shuffle=True,
pin_memory=True)
# train
ac_list, loss_list = train(clf, domain_clf, optimizer, EP,
src_train_loader, tar_train_loader, src, tar)
ac_list = np.array(ac_list).flatten()
# plot tsne
loss_list = np.array(loss_list).flatten()
epoch = [i for i in range(EP)]
my_function.tsne_plot(clf, src_train_loader, tar_train_loader, src, tar,
BATCH_SIZE, 'reverse_grad')
### plot learning curve ###
plt.figure()
plt.plot(epoch, ac_list)
plt.xlabel('EPOCH')
plt.ylabel('Accuracy')
plt.title('domian_adapt : ' + src + ' to ' + tar)
plt.savefig('./learning_curve/domian_adapt_' + src + '_to_' + tar +
'_accuracy.jpg')
plt.figure()
plt.plot(epoch, loss_list)
plt.xlabel('EPOCH')
plt.ylabel('Loss')
plt.title('domian_adapt : ' + src + ' to ' + tar)
plt.savefig('./learning_curve/domian_adapt_' + src + '_to_' + tar +
'_loss.jpg')
def train():
# Turn on training mode which enables dropout.
if args.model == 'QRNN': model.reset()
total_loss = 0
start_time = time.time()
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(args.batch_size)
batch, i = 0, 0
while i < train_data.size(0) - 1 - 1:
bptt = args.bptt if np.random.random() < 0.95 else args.bptt / 2.
# Prevent excessively small or negative sequence lengths
seq_len = max(5, int(np.random.normal(bptt, 5)))
# There's a very small chance that it could select a very long sequence length resulting in OOM
seq_len = min(seq_len, args.bptt + 10)
lr2 = optimizer.param_groups[0]['lr']
optimizer.param_groups[0]['lr'] = lr2 * seq_len / args.bptt
model.train()
model_r.train()
model_mlp.train()
data, targets, _ = get_batch(train_data, i, args, seq_len=seq_len)
data_long, _, _ = get_batch(train_data, i, args, seq_len=seq_len)
seq_len_data = data.size(0)
# Starting each batch, we detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
hidden = repackage_hidden(hidden)
optimizer.zero_grad()
output, hidden, rnn_hs, dropped_rnn_hs = model(data,
hidden,
return_h=True)
output = model.decoder(output)
input_emb = model.encoder(data)
# input_emb = model.encoder(data).detach()
# input_emb = model.encoder(data_long)
# input_emb = model.encoder(data_long).detach()
# input_emb_nhid = model_mlp(input_emb)
attention, seq_len_data, reg_len = model_r(input_emb, seq_len_data)
span_emb = (input_emb.unsqueeze(0) * attention).sum(1)
# span_emb = (input_emb_nhid.unsqueeze(0) * attention).sum(1)
span_emb = model_mlp(span_emb)
raw_loss = criterion(output.view(-1, ntokens), targets)
loss = raw_loss
# Activiation Regularization
loss = loss + sum(args.alpha * dropped_rnn_h.pow(2).mean()
for dropped_rnn_h in dropped_rnn_hs[-1:])
# Temporal Activation Regularization (slowness)
loss = loss + sum(args.beta * (rnn_h[1:] - rnn_h[:-1]).pow(2).mean()
for rnn_h in rnn_hs[-1:])
context_emb = dropped_rnn_hs[-2][:seq_len_data]
if args.ns:
span_emb_t = span_emb.transpose(0, 1)
pos_loss = (1 - (context_emb * span_emb).sum(2).sigmoid()).mean()
neg_loss = 0
split_idx_batch = int(torch.randint(args.batch_size, []))
# split_idx_batch = int(torch.randint(1, args.batch_size, []))
least_ns_seq = 0
if split_idx_batch == 0:
least_ns_seq = 10 if data.size(0) > 15 else int(
data.size(0) / 2)
split_idx_seq = int(torch.randint(least_ns_seq, data.size(0), []))
for j in range(1):
span_emb_neg = torch.cat([
span_emb_t[split_idx_batch:], span_emb_t[:split_idx_batch]
], 0).transpose(0, 1)
span_emb_neg = torch.cat([
span_emb_neg[split_idx_seq:], span_emb_neg[:split_idx_seq]
], 0)
neg_loss += (context_emb *
span_emb_neg).sum(2).sigmoid().mean()
# split_idx_batch = int(torch.randint(args.batch_size, []))
# split_idx_batch = int(torch.randint(1, args.batch_size, []))
# least_ns_seq = 0
# if split_idx_batch == 0:
# least_ns_seq = 10 if data.size(0) > 15 else int(data.size(0) / 2)
# split_idx_seq = int(torch.randint(least_ns_seq, data.size(0), []))
loss += args.theta * (pos_loss + neg_loss) # + 1e-6 * reg_len
else:
loss = loss + args.theta * (context_emb - span_emb).pow(2).mean()
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
optimizer.step()
total_loss += raw_loss.data
optimizer.param_groups[0]['lr'] = lr2
if batch % args.log_interval == 0 and batch > 0:
cur_loss = total_loss[0] / args.log_interval
elapsed = time.time() - start_time
print(
'| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | '
'loss {:5.2f} | ppl {:8.2f}'.format(
epoch, batch,
len(train_data) // args.bptt,
optimizer.param_groups[0]['lr'],
elapsed * 1000 / args.log_interval, cur_loss,
math.exp(cur_loss)))
total_loss = 0
start_time = time.time()
###
batch += 1
i += seq_len
def eval():
data_dir = '../data'
vocabulary, word_embeddings, word_to_index_map, index_to_word_map = load_embed(
data_dir + '/wordvec.txt')
training_set = load_data(data_dir + '/train.tsv',
word_to_index_map) # subset for faster test
train_iter = batch_iter(training_set, batch_size)
dev_set = load_data(data_dir + '/dev.tsv', word_to_index_map)
dev_iter = batch_iter(dev_set, 10)
test_set = load_data(data_dir + '/test.tsv', word_to_index_map)
test_iter = batch_iter(test_set, 10)
use_cuda = torch.cuda.is_available()
input_encoder = model.encoder(word_embeddings.shape[0],
embedding_size=300,
hidden_size=hidden_size,
para_init=0.01,
padding_index=1)
input_encoder.embedding.weight.requires_grad = False
inter_atten = model.atten(hidden_size=hidden_size,
label_size=1,
para_init=0.01)
if resume:
input_encoder.load_state_dict(torch.load(path_encoder))
inter_atten.load_state_dict(torch.load(path_atten))
print('model resumed from', path_encoder, 'and', path_atten)
else:
input_encoder.embedding.weight.data.copy_(
torch.from_numpy(word_embeddings))
if use_cuda:
input_encoder.cuda()
inter_atten.cuda()
para1 = list(filter(lambda p: p.requires_grad, input_encoder.parameters()))
para2 = list(inter_atten.parameters())
grad_norm = 0.
para_norm = 0.
for m in input_encoder.modules():
if isinstance(m, nn.Linear):
para_norm += m.weight.data.norm()**2
if m.bias is not None:
para_norm += m.bias.data.norm()**2
for m in inter_atten.modules():
if isinstance(m, nn.Linear):
para_norm += m.weight.data.norm()**2
if m.bias is not None:
para_norm += m.bias.data.norm()**2
para_norm**0.5
train_acc = test_model(train_iter,
input_encoder,
inter_atten,
use_cuda,
threshould=threshould)
dev_acc = test_model(dev_iter,
input_encoder,
inter_atten,
use_cuda,
threshould=threshould)
test_acc = test_model(test_iter,
input_encoder,
inter_atten,
use_cuda,
threshould=threshould)
print('train-acc %.3f, dev-acc %.3f, test-acc %.3f, para-norm %.3f' %
(train_acc, dev_acc, test_acc, para_norm))
os.system('cp *.py %s' % opt.experiment)
if torch.cuda.is_available() and opt.noCuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
# Initialize image batch
imBatch = Variable(torch.FloatTensor(opt.batchSize, 3, 300, 300))
labelBatch = Variable(
torch.FloatTensor(opt.batchSize, opt.numClasses, 300, 300))
maskBatch = Variable(torch.FloatTensor(opt.batchSize, 1, 300, 300))
labelIndexBatch = Variable(torch.LongTensor(opt.batchSize, 1, 300, 300))
# Initialize network
encoder_normal = model.encoder()
decoder_normal = model.decoder()
model_root_normal = '/datasets/cse152-252-sp20-public/unet_checkpoints/unet_original_zq'
epoch_id_normal = 181
encoder_normal.load_state_dict(
torch.load('%s/encoder_%d.pth' % (model_root_normal, epoch_id_normal)))
decoder_normal.load_state_dict(
torch.load('%s/decoder_%d.pth' % (model_root_normal, epoch_id_normal)))
encoder_normal = encoder_normal.eval()
decoder_normal = decoder_normal.eval()
encoder_dilation = model.encoderDilation()
decoder_dilation = model.decoderDilation()
model_root_dilation = '/datasets/cse152-252-sp20-public/unet_checkpoints/unet_original_zq_dilation'
epoch_id_dilation = 180
encoder_dilation.load_state_dict(
# Parameters
reconstruction_loss_weight = 40.0
# ======================================
# Create Netrowks
# ======================================
images = tf.placeholder(tf.float32, (None, 32, 32, 1))
x_real = (images / 255.0) * 2 - 1
# --------------------------------------
# Encoder
with tf.variable_scope('encoder', reuse=tf.AUTO_REUSE):
z_encoded = encoder(x_real)
# --------------------------------------
# Generator
with tf.variable_scope('generator', reuse=tf.AUTO_REUSE):
x_autoencoded = generator(z_encoded)
# sampled z
z_prior = tf.placeholder(tf.float32, (None, 128))
with tf.variable_scope('generator', reuse=tf.AUTO_REUSE):
x_generated = generator(z_prior)
# --------------------------------------
# Discriminator
def main():
place = fluid.CUDAPlace(0) if InferTaskConfig.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
encoder_program = fluid.Program()
with fluid.program_guard(main_program=encoder_program):
enc_output = encoder(ModelHyperParams.src_vocab_size,
ModelHyperParams.max_length + 1,
ModelHyperParams.n_layer, ModelHyperParams.n_head,
ModelHyperParams.d_key, ModelHyperParams.d_value,
ModelHyperParams.d_model,
ModelHyperParams.d_inner_hid,
ModelHyperParams.dropout)
decoder_program = fluid.Program()
with fluid.program_guard(main_program=decoder_program):
predict = decoder(ModelHyperParams.trg_vocab_size,
ModelHyperParams.max_length + 1,
ModelHyperParams.n_layer, ModelHyperParams.n_head,
ModelHyperParams.d_key, ModelHyperParams.d_value,
ModelHyperParams.d_model,
ModelHyperParams.d_inner_hid,
ModelHyperParams.dropout)
# Load model parameters of encoder and decoder separately from the saved
# transformer model.
encoder_var_names = []
for op in encoder_program.block(0).ops:
encoder_var_names += op.input_arg_names
encoder_param_names = filter(
lambda var_name: isinstance(
encoder_program.block(0).var(var_name), fluid.framework.Parameter),
encoder_var_names)
encoder_params = map(encoder_program.block(0).var, encoder_param_names)
decoder_var_names = []
for op in decoder_program.block(0).ops:
decoder_var_names += op.input_arg_names
decoder_param_names = filter(
lambda var_name: isinstance(
decoder_program.block(0).var(var_name), fluid.framework.Parameter),
decoder_var_names)
decoder_params = map(decoder_program.block(0).var, decoder_param_names)
fluid.io.load_vars(exe, InferTaskConfig.model_path, vars=encoder_params)
fluid.io.load_vars(exe, InferTaskConfig.model_path, vars=decoder_params)
# This is used here to set dropout to the test mode.
encoder_program = encoder_program.clone(for_test=True)
decoder_program = decoder_program.clone(for_test=True)
test_data = paddle.batch(paddle.dataset.wmt16.test(
ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size),
batch_size=InferTaskConfig.batch_size)
trg_idx2word = paddle.dataset.wmt16.get_dict(
"de", dict_size=ModelHyperParams.trg_vocab_size, reverse=True)
def post_process_seq(seq,
bos_idx=ModelHyperParams.bos_idx,
eos_idx=ModelHyperParams.eos_idx,
output_bos=InferTaskConfig.output_bos,
output_eos=InferTaskConfig.output_eos):
"""
Post-process the beam-search decoded sequence. Truncate from the first
<eos> and remove the <bos> and <eos> tokens currently.
"""
eos_pos = len(seq) - 1
for i, idx in enumerate(seq):
if idx == eos_idx:
eos_pos = i
break
seq = seq[:eos_pos + 1]
return filter(
lambda idx: (output_bos or idx != bos_idx) and \
(output_eos or idx != eos_idx),
seq)
for batch_id, data in enumerate(test_data()):
batch_seqs, batch_scores = translate_batch(
exe,
[item[0] for item in data],
encoder_program,
encoder_data_input_fields + encoder_util_input_fields,
[enc_output.name],
decoder_program,
decoder_data_input_fields[:-1] + decoder_util_input_fields +
(decoder_data_input_fields[-1], ),
[predict.name],
InferTaskConfig.beam_size,
InferTaskConfig.max_length,
InferTaskConfig.n_best,
len(data),
ModelHyperParams.n_head,
ModelHyperParams.d_model,
ModelHyperParams.eos_idx, # Use eos_idx to pad.
ModelHyperParams.eos_idx, # Use eos_idx to pad.
ModelHyperParams.bos_idx,
ModelHyperParams.eos_idx,
ModelHyperParams.unk_idx,
output_unk=InferTaskConfig.output_unk)
for i in range(len(batch_seqs)):
# Post-process the beam-search decoded sequences.
seqs = map(post_process_seq, batch_seqs[i])
scores = batch_scores[i]
for seq in seqs:
print(" ".join([trg_idx2word[idx] for idx in seq]))
height=opt.height,
width=opt.width,
keep_ratio=opt.keep_ratio))
val_dataset = dataset.listDataset(list_file=opt.valList,
transform=dataset.resizeNormalize(
(opt.width, opt.height)))
nclass = len(alphabet) + 3 # decoder的时候，需要的类别数,3 for SOS,EOS和blank
nc = 1
converter = utils.strLabelConverterForAttention(alphabet)
image = torch.FloatTensor(opt.batchSize, 3, opt.width, opt.height)
criterion = torch.nn.NLLLoss() # 最后的输出要为log_softmax
encoder = model.encoder(opt.height, nc=nc, nh=256)
decoder = model.decoder(nh=256, nclass=nclass, dropout_p=0.1)
# continue training or use the pretrained model to initial the parameters of the encoder and decoder
encoder.apply(weights_init)
decoder.apply(weights_init)
if opt.encoder:
print('loading pretrained encoder model from %s' % opt.encoder)
encoder.load_state_dict(torch.load(opt.encoder))
if opt.decoder:
print('loading pretrained decoder model from %s' % opt.decoder)
decoder.load_state_dict(torch.load(opt.decoder))
if opt.loadModelEpoch > 0:
encoder_path = 'model/encoder_%d.pth' % opt.loadModelEpoch
print('loading pretrained encoder model from %s' % encoder_path)
encoder.load_state_dict(torch.load(encoder_path))
def main(src, tar):
src_clf = encoder().to(device)
src_clf.load_state_dict(
torch.load('./model/reverse_grad_' + src + '2' + tar + '.pth'))
src_clf.eval()
tar_clf = encoder().to(device)
domain_clf = domain_classifier(2).to(device)
optimizer_domain = optim.Adam(domain_clf.parameters(), lr=1e-4)
optimizer_tar = optim.Adam(tar_clf.parameters(), lr=1e-4)
### dataloader ###
if src == 'mnist':
src_train_set = dset.MNIST('./dataset/mnist',
train=True,
download=True,
transform=gray2rgb_transform)
elif src == 'mnistm':
src_train_set = DATASET('./dataset/mnistm/train',
'./dataset/mnistm/train.csv',
transforms=rgb_transform)
elif src == 'svhn':
src_train_set = dset.SVHN(root='./dataset/svhn/',
download=download,
transform=rgb_transform)
if tar == 'svhn':
tar_train_set = dset.SVHN(root='./dataset/svhn/',
download=download,
transform=rgb_transform)
elif tar == 'mnist':
tar_train_set = dset.MNIST('./dataset/mnist',
train=True,
download=True,
transform=gray2rgb_transform)
elif tar == 'mnistm':
src_train_set = DATASET('./dataset/mnistm/train',
'./dataset/mnistm/train.csv',
transform=rgb_transform)
src_train_loader = torch.utils.data.DataLoader(
dataset=src_train_set,
batch_size=BATCH_SIZE,
shuffle=True,
)
tar_train_loader = torch.utils.data.DataLoader(
dataset=tar_train_set,
batch_size=BATCH_SIZE,
shuffle=True,
)
# train
ac_list, loss_list = train(src_clf, tar_clf, domain_clf, optimizer_domain,
optimizer_tar, 50, src_train_loader,
tar_train_loader, src, tar)
ac_list = np.array(ac_list)
# plot tsne
loss_list = np.array(loss_list)
epoch = [i for i in range(EP)]
my_function.tsne_plot(clf, src_train_loader, tar_train_loader, src, tar,
BATCH_SIZE, 'adda_')
### plot learning curve ###
plt.plot(ac_list, epoch)
plt.xlabel('EPOCH')
plt.ylabel('Accuracy')
plt.title('domian_adapt : ' + src + ' to ' + tar)
plt.savefig('./learning_curve/adda_' + src + '_to_' + tar +
'_accuracy.jpg')
plt.plot(loss_list, epoch)
plt.xlabel('EPOCH')
plt.ylabel('Loss')
plt.title('domian_adapt : ' + src + ' to ' + tar)
plt.savefig('./learning_curve/adda_' + src + '_to_' + tar + '_loss.jpg')
def main():
## prepare data
# load MNIST and MNIST-M
(m_train, m_train_y), (m_test, m_test_y) = tf.keras.datasets.mnist.load_data()
mm = pkl.load(open('data/mnistm_data.pkl', 'rb'))
mm_train, mm_train_y = mm['train'], mm['train_label']
# # keep numbers 0-4 in MNIST as content, and numbers 5-9 in MNIST-M as style
# content_image = m_train[m_train_y < 5, ...]
# content_image_y = m_train_y[m_train_y < 5]
# content_image = resize_image(content_image, size=(32, 32))
# content_image = np.repeat(content_image[..., np.newaxis], 3, axis=-1)
# test_content_image = m_train[m_train_y >= 5, ...]
# style_image = mm_train[mm_train_y >= 5, ...]
# style_image_y = mm_train[mm_train_y >= 5]
# style_image, style_image_y = generate_few_shot_style(style_image, style_image_y, num_sample=5)
# style_image = resize_image(style_image, size=(32, 32))
# use all train data in MNIST as content image, and all train data in MNIST-M as style image
content_image = resize_image(m_train, size=(32, 32))
content_image = np.repeat(content_image[..., np.newaxis], 3, axis=-1)
test_content_image = m_test
style_image = resize_image(mm_train, size=(32, 32))
## prepare model
# inputs placeholder
c_img = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
s_img = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
# establish model
c_encode, _ = encoder(c_img)
s_encode, s_layers = encoder(s_img, reuse=True)
c_adain_encode = adain(c_encode, s_encode)
styled_img = decoder(c_adain_encode)
styled_encode, styled_layers = encoder(styled_img, reuse=True)
# loss
content_loss = compute_content_loss(styled_encode, c_adain_encode)
style_loss = compute_style_loss(styled_layers, s_layers)
total_loss = content_loss + 0.01 * style_loss
# optimizer
optimizer = tf.train.AdamOptimizer(1e-4)
train_op = optimizer.minimize(total_loss)
model_summary()
## training
init = tf.global_variables_initializer()
with tf.Session() as sess:
# Creates a file writer for the log directory.
logdir = "logs/"
file_writer = tf.summary.FileWriter(logdir, sess.graph)
# store variables
tf.summary.image("Content image", c_img, max_outputs=10)
tf.summary.image("Style image", s_img, max_outputs=10)
tf.summary.image("Styled image", styled_img, max_outputs=10)
tf.summary.scalar("Content loss", content_loss)
tf.summary.scalar("Style loss", style_loss)
tf.summary.scalar("Total loss", total_loss)
merged = tf.summary.merge_all()
sess.run(init)
# total number of data
num_data = content_image.shape[0]
batch_size = 8
num_batch = num_data // batch_size
for i_episode in range(EPISODE):
# shuffle data
np.random.shuffle(content_image)
np.random.shuffle(style_image)
for i_batch in range(num_batch):
# get a batch of content
c_image = content_image[i_batch*batch_size: (i_batch+1)*batch_size, ...]
c_image = c_image / 255
# random sample a batch of style
idx = np.random.choice(style_image.shape[0], batch_size, replace=False)
s_image = style_image[idx, ...]
s_image = s_image / 255
# training
_, train_loss = sess.run([train_op, total_loss], feed_dict={
c_img: c_image,
s_img: s_image
})
if i_batch % 100 == 0:
# evaluation on test content image
np.random.shuffle(test_content_image)
test_c_image = test_content_image[:10, ...]
test_c_image = resize_image(test_c_image, size=(32, 32))
test_c_image = np.repeat(test_c_image[..., np.newaxis], 3, axis=-1)
test_c_image = test_c_image / 255
test_s_image = style_image[:10, ...] / 255
summary, test_loss = sess.run([merged, total_loss], feed_dict={
c_img: test_c_image,
s_img: test_s_image
})
# log all variables
#num_iter = i_episode * num_batch + i_batch
file_writer.add_summary(summary, global_step=i_episode * num_batch + i_batch)
print('Episode: %d, batch: %d, training cost: %g, test cost: %g' %
(i_episode, i_batch, train_loss, test_loss))
file_writer.close()
###############################################################################
# Load the model
###############################################################################
with open(args.model_file, 'rb') as f:
if args.cuda:
model = torch.load(f).to(device)
else:
model = torch.load(f, map_location='cpu')
if args.cuda and (not args.single) and (torch.cuda.device_count() > 1):
# If applicable, use multi-gpu for training
# Scatters minibatches (in dim=1) across available GPUs
model = nn.DataParallel(model, dim=1)
if isinstance(model, torch.nn.DataParallel):
# if multi-gpu, access real model for training
model = model.module
# after load the rnn params are not a continuous chunk of memory
# this makes them a continuous chunk, and will speed up forward pass
model.rnn.flatten_parameters()
#print(model.encoder(torch.LongTensor([w for w in range(model.encoder.num_embeddings)])))
for idx, embed in enumerate(
model.encoder(
torch.LongTensor([w for w in range(model.encoder.num_embeddings)
])).data.numpy().tolist()):
word = idx2word[idx]
print(word + ' ' + ' '.join(str(f) for f in embed))
#Placeholder
X = tf.placeholder(tf.float32,[None,img_height,img_width,num_channels]);
Y = tf.placeholder(tf.float32,[None,n_outputs]);
def prior_z(latent_dim):
z_mean = tf.zeros(latent_dim);
z_var = tf.ones(latent_dim);
return tfd.MultivariateNormalDiag(z_mean,z_var);
prior_dist = prior_z(z_dim);
z_prime = prior_dist.sample(batch_size);
# z_hat ~ Q(Z|X) variational distribution parametrized by encoder network
z_hat = encoder(X);
z_hat_test = encoder(X,isTrainable=False,reuse=True);
x_hat = generator(z_hat);
x_hat_test = generator(z_hat_test,isTrainable=False,reuse=True);
x_prime = generator(z_prime,reuse=True);
z_hat_logits = code_discriminator(z_hat);
z_prime_logits = code_discriminator(z_prime,reuse=True);
lamda = opts['lamda'];
l2_recons_loss = 0.05*tf.reduce_mean(tf.pow(X - x_hat,2));
# encoder_loss = lamda_enc*l1_recons_loss + RC_w(z_hat_logits);
# generator_loss = lamda_gen*l1_recons_loss + RD_phi(x_hat_logits) + RD_phi(x_prime_logits);
def train(epoch):
# model is a class that inherits nn.Module
# This puts the model in train mode as opposed to eval mode, so it knows which one to use.
print("check 5")
model.encoder.train()
#print(" check lalala")
model.decoder.train()
print("check 6")
# print(model.fc)
# For each batch of training images,
cum_train_loss = 0
cum_val_loss = 0
for batch_idx, batch in enumerate(train_loader):
# Read images and their target labels in the current batch.
images, captions, lengths = Variable(batch[0]), Variable(
batch[1]), batch[2]
targets = pack_padded_sequence(captions, lengths, batch_first=True)[0]
# Load the current training example in the CUDA core if available.
if args.cuda:
images = images.cuda()
features = model.encoder(images)
output = model.decoder(features, captions, lengths)
criterion = torch.nn.CrossEntropyLoss()
loss = criterion(output, targets)
model.decoder.zero_grad()
model.encoder.zero_grad()
loss.backward()
optimizer.step()
pass
cum_train_loss += loss
# Print out the loss and accuracy on the first 10 batches of the validation set.
# adjusting the printing frequency by changing --log-interval option in the command-line.
if batch_idx % args.log_interval == 0:
# Compute the average validation loss and accuracy.
val_loss = evaluate('val', n_batches=10)
# Compute the training loss.
train_loss = loss.data.item()
# Compute the number of examples in this batch.
examples_this_epoch = batch_idx * len(images)
# Compute the progress rate in terms of the batch.
epoch_progress = 100. * batch_idx / len(train_loader)
# Print out the training loss, validation loss, and accuracy with epoch information.
print('Train Epoch: {} [{}/{} ({:.0f}%)]\t'
'Train Loss: {:.6f}\tVal Loss:{:.6f}\t'.format(
epoch, examples_this_epoch, len(train_loader.dataset),
epoch_progress, train_loss, val_loss))
cum_val_loss += val_loss
avg_val_loss = cum_val_loss / (batch_idx + 1)
avg_train_loss = cum_train_loss / (batch_idx + 1)
print('Train Epoch: {}\t'
'Avg Train Loss: {:.6f}\t Val Loss:{:.6f}\t'.format(
epoch, avg_train_loss, avg_val_loss))
# get data
xtr, ytr, xte, yte = mnist_1000(args.mnist_path)
# placeholders
x = tf.placeholder(tf.float32, [None, 784])
n_train_batches = int(1000/args.batch_size)
n_test_batches = int(1000/args.batch_size)
# models
net = autoencoder(x, args.zdim, True) # train
tnet = autoencoder(x, args.zdim, False, reuse=True) # test
# for visualization
z = tf.placeholder(tf.float32, [None, args.zdim])
tennet = encoder(x, args.zdim, reuse=True) # test encoder
tdenet = decoder(z, reuse=True) # test decoder
def train():
loss = -net['elbo'] # negative ELBO
global_step = tf.train.get_or_create_global_step()
lr = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
[int(n_train_batches*args.n_epochs/2)], [1e-3, 1e-4])
train_op = tf.train.AdamOptimizer(lr).minimize(loss,
global_step=global_step)
saver = tf.train.Saver(net['weights'])
logfile = open(os.path.join(savedir, 'train.log'), 'w')
sess = tf.Session()
def train(encoder, cls_model_1, cls_model_2, optimizer_encoder, optimizer_clf_1, optimizer_clf_2, ep, train_loader, test_loader, src_name, tar_name):
loss_fn_cls = nn.CrossEntropyLoss()
ac_list, loss_list = [], []
max_= 0
end = time.time()
for i in range(ep):
cls_model_1.train()
cls_model_2.train()
encoder.train()
print(i)
for index, (src_batch, tar_batch) in enumerate(zip(train_loader, test_loader)):
# step 1
x, y = src_batch
x = x.to(device)
y = y.to(device)
y = y.view(-1)
feature = encoder(x)
pred1 = cls_model_1(feature)
pred2 = cls_model_2(feature)
loss1 = loss_fn_cls(pred1, y)
loss2 = loss_fn_cls(pred2, y)
loss = loss1 + loss2
optimizer_encoder.zero_grad()
optimizer_clf_1.zero_grad()
optimizer_clf_2.zero_grad()
loss.backward()
optimizer_encoder.step()
optimizer_clf_1.step()
optimizer_clf_2.step()
optimizer_encoder.zero_grad()
optimizer_clf_1.zero_grad()
optimizer_clf_2.zero_grad()
# step 2
tar_x, _ = tar_batch
tar_x = tar_x.to(device)
s_feature = encoder(x)
pred_s_1 = cls_model_1(s_feature)
pred_s_2 = cls_model_2(s_feature)
t_feature = encoder(tar_x)
pred_tar_1 = cls_model_1(t_feature)
pred_tar_2 = cls_model_2(t_feature)
src_loss = loss_fn_cls(pred_s_1, y) + loss_fn_cls(pred_s_2, y)
discrepency_loss = torch.mean(torch.abs(F.softmax(pred_tar_1, dim=1) - F.softmax(pred_tar_2, dim=1)))
loss = src_loss - discrepency_loss
optimizer_clf_1.zero_grad()
optimizer_clf_2.zero_grad()
loss.backward()
optimizer_clf_1.step()
optimizer_clf_2.step()
optimizer_encoder.zero_grad()
optimizer_clf_1.zero_grad()
optimizer_clf_2.zero_grad()
# step 3
for i in range(3):
t_feature = encoder(tar_x)
pred_tar_1 = cls_model_1(t_feature)
pred_tar_2 = cls_model_2(t_feature)
discrepency_loss = torch.mean(abs(F.softmax(pred_tar_1, dim=1) - F.softmax(pred_tar_2, dim=1)))
discrepency_loss.backward()
optimizer_encoder.step()
optimizer_encoder.zero_grad()
optimizer_clf_1.zero_grad()
optimizer_clf_2.zero_grad()
if index % 100 == 0:
print('[%d]/[%d]' % (index, min([len(train_loader), len(test_loader)])))
cls_model_1.eval()
cls_model_2.eval()
encoder.eval()
ac_1 = 0
ac_2 = 0
ac_3 = 0
total_loss=0
with torch.no_grad():
for batch in test_loader:
x, y = batch
x = x.to(device)
y = y.to(device)
y = y.view(-1)
feature = encoder(x)
pred_c1 = cls_model_1(feature)
pred_c2 = cls_model_2(feature)
pred_combine = pred_c1 + pred_c2
ac_1 += np.sum(np.argmax(pred_c1.cpu().detach().numpy(), axis=1) == y.cpu().detach().numpy())
ac_2 += np.sum(np.argmax(pred_c2.cpu().detach().numpy(), axis=1) == y.cpu().detach().numpy())
ac_3 += np.sum(np.argmax(pred_combine.cpu().detach().numpy(), axis=1) == y.cpu().detach().numpy())
total_loss += loss.item()
print('Accuracy : [%.3f], Avg Loss : [%.4f]' % ((ac_1 / len(test_loader) / BATCH_SIZE), (total_loss / len(test_loader))) )
print('Accuracy : [%.3f], Avg Loss : [%.4f]' % ((ac_2 / len(test_loader) / BATCH_SIZE), (total_loss / len(test_loader))) )
print('Accuracy : [%.3f], Avg Loss : [%.4f]' % ((ac_3 / len(test_loader) / BATCH_SIZE), (total_loss / len(test_loader))) )
ac = max([ac_1, ac_2, ac_3])
ac_list.append(ac/len(test_loader)/BATCH_SIZE)
loss_list.append(total_loss / len(test_loader) / BATCH_SIZE)
if (ac / len(test_loader) / BATCH_SIZE) > max_:
max_ = (ac / len(test_loader) / BATCH_SIZE)
torch.save(cls_model_1.state_dict(), './model/mcd_'+src_name+'2'+tar_name+'_1.pth')
torch.save(cls_model_2.state_dict(), './model/mcd_'+src_name+'2'+tar_name+'_2.pth')
torch.save(encoder.state_dict(), './model/mcd_'+src_name+'2'+tar_name+'.pth')
return ac_list, loss_list
# Initialize image batch
imBatch = Variable(torch.FloatTensor(opt.batchSize, 3, 300, 300) )
labelBatch = Variable(torch.FloatTensor(opt.batchSize, opt.numClasses, 300, 300) )
maskBatch = Variable(torch.FloatTensor(opt.batchSize, 1, 300, 300) )
labelIndexBatch = Variable(torch.LongTensor(opt.batchSize, 1, 300, 300) )
# Initialize network
if opt.isDilation:
encoder = model.encoderDilation()
decoder = model.decoderDilation()
elif opt.isSpp:
encoder = model.encoderSPP()
decoder = model.decoderSPP()
else:
encoder = model.encoder()
decoder = model.decoder()
encoder.load_state_dict(torch.load('%s/encoder_%d.pth' % (opt.modelRoot, opt.epochId) ) )
decoder.load_state_dict(torch.load('%s/decoder_%d.pth' % (opt.modelRoot, opt.epochId) ) )
encoder = encoder.eval()
decoder = decoder.eval()
# Move network and containers to gpu
if not opt.noCuda:
imBatch = imBatch.cuda(opt.gpuId )
labelBatch = labelBatch.cuda(opt.gpuId )
labelIndexBatch = labelIndexBatch.cuda(opt.gpuId )
maskBatch = maskBatch.cuda(opt.gpuId )
encoder = encoder.cuda(opt.gpuId )
decoder = decoder.cuda(opt.gpuId )
def py_infer(test_data, trg_idx2word, use_wordpiece):
"""
Inference by beam search implented by python, while the calculations from
symbols to probilities execute by Fluid operators.
"""
place = fluid.CUDAPlace(0) if InferTaskConfig.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
encoder_program = fluid.Program()
with fluid.program_guard(main_program=encoder_program):
enc_output = encoder(
ModelHyperParams.src_vocab_size, ModelHyperParams.max_length + 1,
ModelHyperParams.n_layer, ModelHyperParams.n_head,
ModelHyperParams.d_key, ModelHyperParams.d_value,
ModelHyperParams.d_model, ModelHyperParams.d_inner_hid,
ModelHyperParams.dropout, ModelHyperParams.weight_sharing)
decoder_program = fluid.Program()
with fluid.program_guard(main_program=decoder_program):
predict = decoder(
ModelHyperParams.trg_vocab_size, ModelHyperParams.max_length + 1,
ModelHyperParams.n_layer, ModelHyperParams.n_head,
ModelHyperParams.d_key, ModelHyperParams.d_value,
ModelHyperParams.d_model, ModelHyperParams.d_inner_hid,
ModelHyperParams.dropout, ModelHyperParams.weight_sharing)
# Load model parameters of encoder and decoder separately from the saved
# transformer model.
encoder_var_names = []
for op in encoder_program.block(0).ops:
encoder_var_names += op.input_arg_names
encoder_param_names = filter(
lambda var_name: isinstance(
encoder_program.block(0).var(var_name), fluid.framework.Parameter),
encoder_var_names)
encoder_params = map(encoder_program.block(0).var, encoder_param_names)
decoder_var_names = []
for op in decoder_program.block(0).ops:
decoder_var_names += op.input_arg_names
decoder_param_names = filter(
lambda var_name: isinstance(
decoder_program.block(0).var(var_name), fluid.framework.Parameter),
decoder_var_names)
decoder_params = map(decoder_program.block(0).var, decoder_param_names)
fluid.io.load_vars(exe, InferTaskConfig.model_path, vars=encoder_params)
fluid.io.load_vars(exe, InferTaskConfig.model_path, vars=decoder_params)
# This is used here to set dropout to the test mode.
encoder_program = encoder_program.inference_optimize()
decoder_program = decoder_program.inference_optimize()
for batch_id, data in enumerate(test_data.batch_generator()):
batch_seqs, batch_scores = translate_batch(
exe,
[item[0] for item in data],
encoder_program,
encoder_data_input_fields + encoder_util_input_fields,
[enc_output.name],
decoder_program,
decoder_data_input_fields[:-1] + decoder_util_input_fields +
(decoder_data_input_fields[-1], ),
[predict.name],
InferTaskConfig.beam_size,
InferTaskConfig.max_out_len,
InferTaskConfig.n_best,
len(data),
ModelHyperParams.n_head,
ModelHyperParams.d_model,
ModelHyperParams.eos_idx, # Use eos_idx to pad.
ModelHyperParams.eos_idx, # Use eos_idx to pad.
ModelHyperParams.bos_idx,
ModelHyperParams.eos_idx,
ModelHyperParams.unk_idx,
output_unk=InferTaskConfig.output_unk)
for i in range(len(batch_seqs)):
# Post-process the beam-search decoded sequences.
seqs = map(post_process_seq, batch_seqs[i])
scores = batch_scores[i]
for seq in seqs:
if use_wordpiece:
print(util.subword_ids_to_str(seq, trg_idx2word))
else:
print(" ".join([trg_idx2word[idx] for idx in seq]))