def inference(self, input_):
self.loader = BatchLoader(
self.config.data_dir, self.config.dataset_name,
self.config.batch_size, self.config.num_classes,
self.config.preprocessor, self.config.epoch,
self.config.specialist, self.config.forward_only)
content, filename = self.loader.prepare_inference()
#with tf.control_dependencies([self.loader.enqueue]):
logits = self.build_model(content)
softmax = tf.nn.softmax(logits)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
self.sess.run(init_op)
self.saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(self.config.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
self.saver.restore(self.sess, ckpt.model_checkpoint_path)
else:
print('no checkpoint found...')
self.sess.run(self.loader.enqueue,
feed_dict={self.loader.filenames: input_})
m_logits, m_softmax, m_filename = self.sess.run(
[logits, softmax, filename])
print(m_softmax, m_filename)
class LandmarkDataLayer(caffe.Layer):
def setup(self, bottom, top):
param = eval(self.param_str)
self.batch = int(param['batch'])
self.img_size = config.IMG_SIZE
self.batch_loader = BatchLoader(param, "train")
self.ohem_batch_loader = BatchLoader(param, 'ohem')
self.train_ratio = 1.
self.ohem_ratio = 0
if self.ohem_batch_loader.is_loaded():
self.train_ratio = 7 / 8.
self.ohem_ratio = 1. - self.train_ratio
top[0].reshape(self.batch, 3, self.img_size, self.img_size) # data
top[1].reshape(self.batch, config.LANDMARK_SIZE * 2) # landmark
top[2].reshape(self.batch, 1) # landmark
def reshape(self, bottom, top):
pass
def forward(self, bottom, top):
batch_data = self.batch_loader.next_batch(
self.batch * self.train_ratio, '')
batch_data += self.ohem_batch_loader.next_batch(
self.batch * self.ohem_ratio, '')
for i, datum in enumerate(batch_data):
img, pts, eye_dist = datum
top[0].data[i, ...] = img
top[1].data[i, ...] = pts
top[2].data[i, ...] = eye_dist
def backward(self, bottom, top):
pass
class LandmarkDataLayer(caffe.Layer):
def setup(self, bottom, top):
param = eval(self.param_str)
self.batch = int(param['batch'])
self.batch_loader = BatchLoader(param, "train")
self.ohem_batch_loader = BatchLoader(param, 'ohem')
self.train_ratio = 1.
self.ohem_ratio = 0
if self.ohem_batch_loader.is_loaded():
self.train_ratio = 7 / 8.
self.ohem_ratio = 1. - self.train_ratio
top[0].reshape(self.batch, 3, self.img_size, self.img_size) # data
top[1].reshape(self.batch, 72) # landmark
def reshape(self, bottom, top):
pass
def forward(self, bottom, top):
batch_data = self.batch_loader.next_batch(
self.batch * self.train_ratio, '')
batch_data += self.ohem_batch_loader.next_batch(
self.batch * self.ohem_ratio, '')
random.shuffle(batch_data)
for i, datum in enumerate(batch_data):
img, label, bbox, landm5 = datum
top[0].data[i, ...] = img
top[1].data[i, ...] = label
top[2].data[i, ...] = bbox
if self.net != 'pnet':
top[3].data[i, ...] = landm5
def backward(self, bottom, top):
pass
def setup(self, bottom, top):
param = eval(self.param_str)
self.batch = int(param['batch'])
self.batch_loader = BatchLoader(param, "train")
self.ohem_batch_loader = BatchLoader(param, 'ohem')
self.train_ratio = 1.
self.ohem_ratio = 0
if self.ohem_batch_loader.is_loaded():
self.train_ratio = 7 / 8.
self.ohem_ratio = 1. - self.train_ratio
top[0].reshape(self.batch, 3, self.img_size, self.img_size) # data
top[1].reshape(self.batch, 72) # landmark
def __init__(self,
batch_size=params.BATCH_SIZE,
nb_epochs=params.NB_EPOCHS,
mask=None,
experiment_path=params.EXPERIMENT_PATH,
use_adversarial_loss=params.USE_ADVERSARIAL_LOSS,
lambda_decay=params.LAMBDA_DECAY,
lambda_adversarial=params.LAMBDA_ADVERSARIAL,
patience=params.PATIENCE,
discr_whole_image=params.DISCR_WHOLE_IMAGE,
discr_loss_limit=params.DISCR_LOSS_LIMIT,
use_dropout=params.USE_DROPOUT):
self.batch_size = batch_size
self.nb_epochs = nb_epochs
self.experiment_path = experiment_path
self.save_path = os.path.join(self.experiment_path, "model/")
self.save_best_path = os.path.join(self.experiment_path, "best_model/")
self.logs_path = os.path.join(self.experiment_path, "logs")
create_dir(self.save_path)
create_dir(self.logs_path)
self.global_step = tf.Variable(0, trainable=False, name='global_step')
self.phase = tf.placeholder(tf.bool, name='phase')
self.patience = patience
# parameters for adversarial loss
self.use_adversarial_loss = use_adversarial_loss
self.lambda_adversarial = lambda_adversarial
if lambda_decay:
self.lambda_adversarial = 1 - tf.train.exponential_decay(
.1, self.global_step, 10000, .5, staircase=True)
self.discr_whole_image = discr_whole_image
self.discr_loss_limit = discr_loss_limit
self.num_discr_trained = tf.Variable(tf.constant(0, dtype=tf.int32),
trainable=False)
self.use_dropout = use_dropout
self.batch_loader = BatchLoader(self.batch_size)
if mask is not None:
self.np_mask = mask
else:
self.np_mask = np.zeros((1, 64, 64, 1))
self.np_mask[:, 16:48, 16:48, :] = 1
self._sess = tf.Session()
tf.summary.scalar("lambda_adversarial", self.lambda_adversarial)
tf.summary.scalar("num discr trained", self.num_discr_trained)
class GanNetTrain(caffe.Layer):
"""Data layer for training"""
def setup(self, bottom, top):
layer_params = yaml.load(self.param_str)
self.batch_size = layer_params['batch_size']
self.image_file = layer_params['image_file']
self.batch_loader = BatchLoader(self.image_file, self.batch_size)
def forward(self, bottom, top):
# assign output
top[0].data[...] = self.images_a
top[1].data[...] = self.images_b
top[2].data[...] = self.label_true
top[3].data[...] = self.label_false
def backward(self, top, propagate_down, bottom):
"""This layer does not propagate gradients."""
pass
def reshape(self, bottom, top):
images_A, images_B = self.batch_loader.next_batch()
images_A = np.array(images_A)
images_B = np.array(images_B)
images_A = images_A.transpose((0, 3, 1, 2))
images_B = images_B.transpose((0, 3, 1, 2))
self.images_a = images_A
self.images_b = images_B
self.label_true = np.ones((self.batch_size, 1), dtype='float32')
self.label_false = np.zeros((self.batch_size, 1), dtype='float32')
top[0].reshape(*self.images_a.shape)
top[1].reshape(*self.images_b.shape)
top[2].reshape(*self.label_true.shape)
top[3].reshape(*self.label_false.shape)
def pretrain_generator(gen, gen_opt, epochs):
global data_df
n_iter = 0
loader = BatchLoader(data_df)
for epoch in range(epochs):
# print(f'epoch = {epoch}\n --------------------------------')
total_loss = 0
n_iter += 1
for batch in tqdm(loader.load_action_batch(MAX_SEQ_LEN, BATCH_SIZE,
CUDA),
total=int(NUM_SAMPLES / BATCH_SIZE / MAX_SEQ_LEN)):
gen_opt.zero_grad()
loss = gen.batchNLLLoss(batch)
loss.backward()
gen_opt.step()
total_loss += loss.data.item()
total_loss /= NUM_SAMPLES / BATCH_SIZE / MAX_SEQ_LEN
print('iteration = %d, NLL loss = %.4f' % (n_iter, total_loss))
def setup(self, bottom, top):
param = eval(self.param_str)
self.batch = int(param['batch'])
self.net = param['net']
self.img_size = config.NET_IMG_SIZES[self.net]
self.batch_loader = BatchLoader(param, "train")
self.ohem_batch_loader = BatchLoader(param, 'ohem')
self.train_ratio = 1.
self.ohem_ratio = 0
if self.ohem_batch_loader.is_loaded():
self.train_ratio = 7 / 8.
self.ohem_ratio = 1 / 8.
top[0].reshape(self.batch, 3, self.img_size, self.img_size) # data
top[1].reshape(self.batch, 1) # label
top[2].reshape(self.batch, 4) # bbox
if self.net != 'pnet':
top[3].reshape(self.batch, config.LANDMARK_SIZE * 2)
def train(args):
yaml_file = yaml.safe_load(args.cfg)
print(yaml.dump(yaml_file, default_flow_style=False))
cfg = TrainingConfig(yaml_file)
output_dir = cfg.output
if not os.path.exists(output_dir):
os.makedirs(output_dir)
chunks = get_all_chunks(cfg.input)
print("Found {} files".format(len(chunks)))
num_train_chunks = int(len(chunks) * cfg.train_ratio)
training_chunks = chunks[:num_train_chunks]
test_chunks = chunks[num_train_chunks:]
print("Chunks Training({}) Testing({})".format(len(training_chunks),
len(test_chunks)))
train_loader = BatchLoader(training_chunks, cfg)
test_loader = BatchLoader(test_chunks, cfg)
worker = TensorWorker(cfg, train_loader, test_loader)
print()
class DataLayer(caffe.Layer):
def setup(self, bottom, top):
param = eval(self.param_str)
self.batch = int(param['batch'])
self.net = param['net']
self.img_size = config.NET_IMG_SIZES[self.net]
self.batch_loader = BatchLoader(param, "train")
self.ohem_batch_loader = BatchLoader(param, 'ohem')
self.train_ratio = 1.
self.ohem_ratio = 0
if self.ohem_batch_loader.is_loaded():
self.train_ratio = 7 / 8.
self.ohem_ratio = 1 / 8.
top[0].reshape(self.batch, 3, self.img_size, self.img_size) # data
top[1].reshape(self.batch, 1) # label
top[2].reshape(self.batch, 4) # bbox
if self.net != 'pnet':
top[3].reshape(self.batch, config.LANDMARK_SIZE * 2)
def reshape(self, bottom, top):
pass
def forward(self, bottom, top):
task = random.choice(config.TRAIN_TASKS[self.net])
batch_data = self.batch_loader.next_batch(
self.batch * self.train_ratio, '')
batch_data += self.ohem_batch_loader.next_batch(
self.batch * self.ohem_ratio, '')
random.shuffle(batch_data)
for i, datum in enumerate(batch_data):
img, label, bbox, landm5 = datum
top[0].data[i, ...] = img
top[1].data[i, ...] = label
top[2].data[i, ...] = bbox
if self.net != 'pnet':
top[3].data[i, ...] = landm5
def backward(self, bottom, top):
pass
def __init__(self, vocab_files, \
generator_file, \
batch_size=32, load_generator=True):
self.batch_size = batch_size
self.generator_data = None
if generator_file:
with open(generator_file, 'rb') as f:
self.generator_data = [pkl.load(f)]
self.gen_batch_loader = BatchLoader(self.generator_data,
vocab_files,
sentence_array=True)
self.vocab_size = self.gen_batch_loader.words_vocab_size
def main():
np.random.seed(233)
torch.manual_seed(233)
data_loader = BatchLoader(10000)
model = Seq2SeqModel(10, NUM_CLASSES, WINDOW_SIZE)
trainval(model, data_loader)
# Should print 'polo '
inference(model, "nor marco I", "polo ")
inference(model, "marco nor I", "polo ")
inference(model, "nor I marco", "polo ")
# Should print ' '
inference(model, "nor I neither", " ")
# More difficult task
inference(model, "nor ma rco I", " ")
inference(model, "ma rco nor I", " ")
pass
def train(self,
train_image_indices,
batch_size,
num_epochs=50,
train_method='normal',
lambda_1=0,
lambda_2=0,
start_from_pretrained_model=True,
learning_rate=0.01,
optimizer='SGD'):
if os.path.exists(self.checkpoint_path):
os.remove(self.checkpoint_path)
model = self.initialize_model(
start_from_pretrained_model=start_from_pretrained_model)
model = model.to(self.device)
criterion = nn.CrossEntropyLoss()
if optimizer == 'SGD':
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
momentum=0.9,
weight_decay=5e-4)
elif optimizer == 'Adam':
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=5e-4)
else:
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
momentum=0.9,
weight_decay=5e-4)
train_batch_loader = BatchLoader(self.train_folder_path,
train_image_indices)
n_images = len(train_image_indices)
if n_images % batch_size == 0:
num_batches = n_images // batch_size
else:
num_batches = (n_images // batch_size) + 1
penalty_inside_list = []
penalty_outside_list = []
train_acc_list = []
train_loss_list = []
val_loss_list = []
val_acc_list = []
best_acc = 0.0
for epoch in range(num_epochs):
model.train()
train_batch_loader.reset()
print('Epoch: {}/{}'.format(epoch + 1, num_epochs))
print('-' * 50)
train_correct = 0.0
train_loss = 0.0
penalty_inside = 0.0
penalty_outside = 0.0
for batch in range(num_batches):
batch_indices = train_batch_loader.get_batch_indices(
batch_size)
inputs = self.x_train[batch_indices]
labels = self.y_train[batch_indices]
inputs, labels = inputs.to(self.device), labels.to(self.device)
if train_method == 'bbox':
inputs.requires_grad_()
outputs = model(inputs)
preds = torch.argmax(outputs, dim=1)
# cross entropy loss
loss = criterion(outputs, labels)
input_gradient = torch.autograd.grad(loss,
inputs,
create_graph=True)[0]
penalty_inside_box, penalty_outside_box = self.calculate_penalty_box(
batch_indices, input_gradient)
new_loss = loss + lambda_1 * penalty_inside_box + lambda_2 * penalty_outside_box
optimizer.zero_grad()
new_loss.backward()
optimizer.step()
else:
outputs = model(inputs)
preds = torch.argmax(outputs, dim=1)
# cross entropy loss
loss = criterion(outputs, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
penalty_inside_box = torch.tensor(0).to(self.device)
penalty_outside_box = torch.tensor(0).to(self.device)
train_loss += loss.item()
train_correct += torch.sum(preds == labels).float().item()
penalty_inside += penalty_inside_box.item() * lambda_1
penalty_outside += penalty_outside_box.item() * lambda_2
train_loss = train_loss / self.train_dataset_length
train_loss_list.append(train_loss)
train_acc = (train_correct / self.train_dataset_length) * 100.0
train_acc_list.append(train_acc)
penalty_inside = penalty_inside / self.train_dataset_length
penalty_outside = penalty_outside / self.train_dataset_length
penalty_inside_list.append(penalty_inside)
penalty_outside_list.append(penalty_outside)
print('Train Loss: {:.4f} Acc: {:.4f} % '.format(
train_loss, train_acc))
print(f'Penalty Inside Box: {round(penalty_inside, 4)}')
print(f'Penalty Outside Box: {round(penalty_outside, 4)}')
# validate after each epoch
val_correct = 0.0
val_loss = 0.0
model.eval()
with torch.no_grad():
for inputs_val, labels_val in self.val_loader:
inputs_val, labels_val = inputs_val.to(
self.device), labels_val.to(self.device)
outputs_val = model(inputs_val)
preds_val = torch.argmax(outputs_val, dim=1)
loss_test = criterion(outputs_val, labels_val)
val_loss += loss_test.item()
val_correct += torch.sum(
preds_val == labels_val).float().item()
val_loss = val_loss / self.val_dataset_length
val_loss_list.append(val_loss)
val_acc = (val_correct / self.val_dataset_length) * 100.0
val_acc_list.append(val_acc)
print('Val Loss: {:.4f} Acc: {:.4f} % \n'.format(
val_loss, val_acc))
# save the best model
if val_acc > best_acc:
best_acc = val_acc
model.state_dict()
if os.path.exists(self.checkpoint_path):
os.remove(self.checkpoint_path)
torch.save(model.state_dict(), self.checkpoint_path)
return_dict = {
'train_acc_list': train_acc_list,
'train_loss_list': train_loss_list,
'penalty_inside_list': penalty_inside_list,
'penalty_outside_list': penalty_outside_list,
'val_loss_list': val_loss_list,
'val_acc_list': val_acc_list,
'best_acc': best_acc
}
return return_dict
# ## Import modules
# In[2]:
import os
import cv2
import numpy as np
import tensorflow as tf
import tflearn
from batch_loader import BatchLoader
slim = tf.contrib.slim
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
train_batch_loader = BatchLoader("./train.txt", 120)
test_batch_loader = BatchLoader("./test.txt", 100)
# ## Construct network
# In[3]:
with tf.name_scope('input'):
input_images = tf.placeholder(tf.float32,
shape=(None, 100, 100, 3),
name='input_images')
labels = tf.placeholder(tf.int64, shape=(None), name='labels')
global_step = tf.Variable(0, trainable=False, name='global_step')
# In[4]:
def __init__(self, sess,
batch_size=100, rnn_size=650, layer_depth=2,
word_embed_dim=650, char_embed_dim=15,
feature_maps=[50, 100, 150, 200, 200, 200, 200],
kernels=[1,2,3,4,5,6,7], seq_length=35, max_word_length=65,
use_word=False, use_char=True, hsm=0, max_grad_norm=5,
highway_layers=2, dropout_prob=0.5, use_batch_norm=True,
checkpoint_dir="checkpoint", forward_only=False,
data_dir="data", dataset_name="pdb", use_progressbar=False):
"""
Initialize the parameters for LSTM TDNN
Args:
rnn_size: the dimensionality of hidden layers
layer_depth: # of depth in LSTM layers
batch_size: size of batch per epoch
word_embed_dim: the dimensionality of word embeddings
char_embed_dim: the dimensionality of character embeddings
feature_maps: list of feature maps (for each kernel width)
kernels: list of kernel widths
seq_length: max length of a word
use_word: whether to use word embeddings or not
use_char: whether to use character embeddings or not
highway_layers: # of highway layers to use
dropout_prob: the probability of dropout
use_batch_norm: whether to use batch normalization or not
hsm: whether to use hierarchical softmax
"""
self.sess = sess
self.batch_size = batch_size
self.seq_length = seq_length
# RNN
self.rnn_size = rnn_size
self.layer_depth = layer_depth
# CNN
self.use_word = use_word
self.use_char = use_char
self.word_embed_dim = word_embed_dim
self.char_embed_dim = char_embed_dim
self.feature_maps = feature_maps
self.kernels = kernels
# General
self.highway_layers = highway_layers
self.dropout_prob = dropout_prob
self.use_batch_norm = use_batch_norm
# Training
self.max_grad_norm = max_grad_norm
self.max_word_length = max_word_length
self.hsm = hsm
self.data_dir = data_dir
self.dataset_name = dataset_name
self.checkpoint_dir = checkpoint_dir
self.forward_only = forward_only
self.use_progressbar = use_progressbar
self.loader = BatchLoader(self.data_dir, self.dataset_name, self.batch_size, self.seq_length, self.max_word_length)
print('Word vocab size: %d, Char vocab size: %d, Max word length (incl. padding): %d' % \
(len(self.loader.idx2word), len(self.loader.idx2char), self.loader.max_word_length))
self.max_word_length = self.loader.max_word_length
self.char_vocab_size = len(self.loader.idx2char)
self.word_vocab_size = len(self.loader.idx2word)
# build LSTMTDNN model
self.prepare_model()
# load checkpoints
if self.forward_only == True:
if self.load(self.checkpoint_dir, self.dataset_name):
print("[*] SUCCESS to load model for %s." % self.dataset_name)
else:
print("[!] Failed to load model for %s." % self.dataset_name)
sys.exit(1)
class LSTMTDNN(Model):
"""
Time-delayed Neural Network (cf. http://arxiv.org/abs/1508.06615v4)
"""
def __init__(self, sess,
batch_size=100, rnn_size=650, layer_depth=2,
word_embed_dim=650, char_embed_dim=15,
feature_maps=[50, 100, 150, 200, 200, 200, 200],
kernels=[1,2,3,4,5,6,7], seq_length=35, max_word_length=65,
use_word=False, use_char=True, hsm=0, max_grad_norm=5,
highway_layers=2, dropout_prob=0.5, use_batch_norm=True,
checkpoint_dir="checkpoint", forward_only=False,
data_dir="data", dataset_name="pdb", use_progressbar=False):
"""
Initialize the parameters for LSTM TDNN
Args:
rnn_size: the dimensionality of hidden layers
layer_depth: # of depth in LSTM layers
batch_size: size of batch per epoch
word_embed_dim: the dimensionality of word embeddings
char_embed_dim: the dimensionality of character embeddings
feature_maps: list of feature maps (for each kernel width)
kernels: list of kernel widths
seq_length: max length of a word
use_word: whether to use word embeddings or not
use_char: whether to use character embeddings or not
highway_layers: # of highway layers to use
dropout_prob: the probability of dropout
use_batch_norm: whether to use batch normalization or not
hsm: whether to use hierarchical softmax
"""
self.sess = sess
self.batch_size = batch_size
self.seq_length = seq_length
# RNN
self.rnn_size = rnn_size
self.layer_depth = layer_depth
# CNN
self.use_word = use_word
self.use_char = use_char
self.word_embed_dim = word_embed_dim
self.char_embed_dim = char_embed_dim
self.feature_maps = feature_maps
self.kernels = kernels
# General
self.highway_layers = highway_layers
self.dropout_prob = dropout_prob
self.use_batch_norm = use_batch_norm
# Training
self.max_grad_norm = max_grad_norm
self.max_word_length = max_word_length
self.hsm = hsm
self.data_dir = data_dir
self.dataset_name = dataset_name
self.checkpoint_dir = checkpoint_dir
self.forward_only = forward_only
self.use_progressbar = use_progressbar
self.loader = BatchLoader(self.data_dir, self.dataset_name, self.batch_size, self.seq_length, self.max_word_length)
print('Word vocab size: %d, Char vocab size: %d, Max word length (incl. padding): %d' % \
(len(self.loader.idx2word), len(self.loader.idx2char), self.loader.max_word_length))
self.max_word_length = self.loader.max_word_length
self.char_vocab_size = len(self.loader.idx2char)
self.word_vocab_size = len(self.loader.idx2word)
# build LSTMTDNN model
self.prepare_model()
# load checkpoints
if self.forward_only == True:
if self.load(self.checkpoint_dir, self.dataset_name):
print("[*] SUCCESS to load model for %s." % self.dataset_name)
else:
print("[!] Failed to load model for %s." % self.dataset_name)
sys.exit(1)
def prepare_model(self):
with tf.variable_scope("LSTMTDNN"):
self.char_inputs = []
self.word_inputs = []
self.cnn_outputs = []
if self.use_char:
char_W = tf.get_variable("char_embed",
[self.char_vocab_size, self.char_embed_dim])
if self.use_word:
word_W = tf.get_variable("word_embed",
[self.word_vocab_size, self.word_embed_dim])
with tf.variable_scope("CNN") as scope:
self.char_inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_length, self.max_word_length])
self.word_inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_length])
char_indices = tf.split(1, self.seq_length, self.char_inputs)
word_indices = tf.split(1, self.seq_length, tf.expand_dims(self.word_inputs, -1))
for idx in xrange(self.seq_length):
char_index = tf.reshape(char_indices[idx], [-1, self.max_word_length])
word_index = tf.reshape(word_indices[idx], [-1, 1])
if idx != 0:
scope.reuse_variables()
if self.use_char:
# [batch_size x word_max_length, char_embed]
char_embed = tf.nn.embedding_lookup(char_W, char_index)
char_cnn = TDNN(char_embed, self.char_embed_dim, self.feature_maps, self.kernels)
if self.use_word:
word_embed = tf.nn.embedding_lookup(word_W, word_index)
cnn_output = tf.concat(1, [char_cnn.output, tf.squeeze(word_embed, [1])])
else:
cnn_output = char_cnn.output
else:
cnn_output = tf.squeeze(tf.nn.embedding_lookup(word_W, word_index))
if self.use_batch_norm:
bn = batch_norm()
norm_output = bn(tf.expand_dims(tf.expand_dims(cnn_output, 1), 1))
cnn_output = tf.squeeze(norm_output)
if highway:
#cnn_output = highway(input_, input_dim_length, self.highway_layers, 0)
cnn_output = highway(cnn_output, cnn_output.get_shape()[1], self.highway_layers, 0)
self.cnn_outputs.append(cnn_output)
with tf.variable_scope("LSTM") as scope:
self.cell = tf.nn.rnn_cell.BasicLSTMCell(self.rnn_size)
self.stacked_cell = tf.nn.rnn_cell.MultiRNNCell([self.cell] * self.layer_depth)
outputs, _ = tf.nn.rnn(self.stacked_cell,
self.cnn_outputs,
dtype=tf.float32)
self.lstm_outputs = []
self.true_outputs = tf.placeholder(tf.int64,
[self.batch_size, self.seq_length])
loss = 0
true_outputs = tf.split(1, self.seq_length, self.true_outputs)
for idx, (top_h, true_output) in enumerate(zip(outputs, true_outputs)):
if self.dropout_prob > 0:
top_h = tf.nn.dropout(top_h, self.dropout_prob)
if self.hsm > 0:
self.lstm_outputs.append(top_h)
else:
if idx != 0:
scope.reuse_variables()
proj = tf.nn.rnn_cell._linear(top_h, self.word_vocab_size, 0)
self.lstm_outputs.append(proj)
loss += tf.nn.sparse_softmax_cross_entropy_with_logits(self.lstm_outputs[idx], tf.squeeze(true_output))
self.loss = tf.reduce_mean(loss) / self.seq_length
tf.scalar_summary("loss", self.loss)
tf.scalar_summary("perplexity", tf.exp(self.loss))
def train(self, epoch):
cost = 0
target = np.zeros([self.batch_size, self.seq_length])
N = self.loader.sizes[0]
for idx in xrange(N):
target.fill(0)
x, y, x_char = self.loader.next_batch(0)
for b in xrange(self.batch_size):
for t, w in enumerate(y[b]):
target[b][t] = w
feed_dict = {
self.word_inputs: x,
self.char_inputs: x_char,
self.true_outputs: target,
}
_, loss, step, summary_str = self.sess.run(
[self.optim, self.loss, self.global_step, self.merged_summary], feed_dict=feed_dict)
self.writer.add_summary(summary_str, step)
if idx % 50 == 0:
if self.use_progressbar:
progress(idx/N, "epoch: [%2d] [%4d/%4d] loss: %2.6f" % (epoch, idx, N, loss))
else:
print("epoch: [%2d] [%4d/%4d] loss: %2.6f" % (epoch, idx, N, loss))
cost += loss
return cost / N
def test(self, split_idx, max_batches=None):
if split_idx == 1:
set_name = 'Valid'
else:
set_name = 'Test'
N = self.loader.sizes[split_idx]
if max_batches != None:
N = min(max_batches, N)
self.loader.reset_batch_pointer(split_idx)
target = np.zeros([self.batch_size, self.seq_length])
cost = 0
for idx in xrange(N):
target.fill(0)
x, y, x_char = self.loader.next_batch(split_idx)
for b in xrange(self.batch_size):
for t, w in enumerate(y[b]):
target[b][t] = w
feed_dict = {
self.word_inputs: x,
self.char_inputs: x_char,
self.true_outputs: target,
}
loss = self.sess.run(self.loss, feed_dict=feed_dict)
if idx % 50 == 0:
if self.use_progressbar:
progress(idx/N, "> %s: loss: %2.6f, perplexity: %2.6f" % (set_name, loss, np.exp(loss)))
else:
print(" > %s: loss: %2.6f, perplexity: %2.6f" % (set_name, loss, np.exp(loss)))
cost += loss
cost = cost / N
return cost
def run(self, epoch=25,
learning_rate=1, learning_rate_decay=0.5):
self.current_lr = learning_rate
self.lr = tf.Variable(learning_rate, trainable=False)
self.opt = tf.train.GradientDescentOptimizer(self.lr)
#self.opt = tf.train.AdamOptimizer(learning_rate, beta1=0.5).minimize(self.loss)
# clip gradients
params = tf.trainable_variables()
grads = []
for grad in tf.gradients(self.loss, params):
if grad is not None:
grads.append(tf.clip_by_norm(grad, self.max_grad_norm))
else:
grads.append(grad)
self.global_step = tf.Variable(0, name="global_step", trainable=False)
self.optim = self.opt.apply_gradients(zip(grads, params),
global_step=self.global_step)
# ready for train
tf.initialize_all_variables().run()
if self.load(self.checkpoint_dir, self.dataset_name):
print("[*] SUCCESS to load model for %s." % self.dataset_name)
else:
print("[!] Failed to load model for %s." % self.dataset_name)
self.saver = tf.train.Saver()
self.merged_summary = tf.merge_all_summaries()
self.writer = tf.train.SummaryWriter("./logs", self.sess.graph_def)
self.log_loss = []
self.log_perp = []
if not self.forward_only:
for idx in xrange(epoch):
train_loss = self.train(idx)
valid_loss = self.test(1)
# Logging
self.log_loss.append([train_loss, valid_loss])
self.log_perp.append([np.exp(train_loss), np.exp(valid_loss)])
state = {
'perplexity': np.exp(train_loss),
'epoch': idx,
'learning_rate': self.current_lr,
'valid_perplexity': np.exp(valid_loss)
}
print(state)
# Learning rate annealing
if len(self.log_loss) > 1 and self.log_loss[idx][1] > self.log_loss[idx-1][1] * 0.9999:
self.current_lr = self.current_lr * learning_rate_decay
self.lr.assign(self.current_lr).eval()
if self.current_lr < 1e-5: break
if idx % 2 == 0:
self.save(self.checkpoint_dir, self.dataset_name)
test_loss = self.test(2)
print("[*] Test loss: %2.6f, perplexity: %2.6f" % (test_loss, np.exp(test_loss)))
def setup(self, bottom, top):
layer_params = yaml.load(self.param_str)
self.batch_size = layer_params['batch_size']
self.image_file = layer_params['image_file']
self.batch_loader = BatchLoader(self.image_file, self.batch_size)
def main():
# setting parameters
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--num_factors',
type=float,
default=10,
help='embedding size')
parser.add_argument('--model', type=str, default='mlp_bpr', help='model')
parser.add_argument('--epoches', type=str, default=1, help='epoches')
parser.add_argument('--learning_rate',
type=float,
default=0.01,
help='learning rate')
parser.add_argument('--reg_lambda',
type=float,
default=1.0,
help='l2_regularizer lambda')
parser.add_argument('--layers',
nargs='?',
default='[10,1]',
help="Size of each layer.")
parser.add_argument('--batch_size',
type=int,
default=10000,
help='minibatch size')
parser.add_argument(
'--recom_mode',
type=str,
default='p_u',
help='recommendation mode, u_p: users to items, p_u: items to users')
parser.add_argument('--decay_rate',
type=float,
default=0.99,
help='decay rate for Adam')
parser.add_argument('--keep_prob',
type=float,
default=0.2,
help='dropout probility')
parser.add_argument(
'--uti_k',
type=int,
default=30,
help='top-k recommendation for recommending items to user')
parser.add_argument(
'--itu_k',
type=int,
default=100,
help='top-k recommendation for recommending users to item')
parser.add_argument('--log_dir',
type=str,
default='logs',
help='directory to store tensorboard logs')
parser.add_argument(
'--mode',
type=str,
default='validation',
help=
'train: only train the model, validation: train the model and test it with test data, predict: predict new data'
)
parser.add_argument(
'--dev',
type=str,
default='cpu',
help=
'training by CPU or GPU, input cpu or gpu:0 or gpu:1 or gpu:2 or gpu:3'
)
parser.add_argument(
'--pIter',
type=int,
default=2,
help='how many rounds of iterations show the effect on the test set')
args = parser.parse_args()
if FLAGS.mode == 'validation':
# read data from file
train_file = 'data/train/train_u.csv'
test_file = 'data/test/test_u.csv'
test_count1_file = 'data/test/test_u_count1.csv'
train_X, train_y, test_X, test_y, test_count1_X, test_count1_y, user_pro_dict, pro_user_dict, user_index_map = utils.read_data_val(
train_file, test_file, test_count1_file)
print(test_count1_X)
# read feature data from file
user_file = 'data/mid/user_features.csv'
item_file = 'data/mid/item_features.csv'
user_feature_df = utils.read_user_data(user_file, user_index_map)
item_feature_df = utils.read_item_data(item_file)
# generate batches
batch_loader = BatchLoader(args.batch_size, train_X, train_y)
args.num_users = np.max([np.max(train_X['uid'])]) + 1
args.num_items = np.max([np.max(train_X['pid'])]) + 1
model = Model(args, args.recom_mode, batch_loader)
model.train_val(test_X, user_pro_dict, pro_user_dict, user_feature_df,
item_feature_df)
#pred_dict = model.test(test_X, user_pro_dict, pro_user_dict, user_feature_df, item_feature_df)
#if args.recom_mode=='u_p':
# print(model.cal_MRR(pred_dict, test_X))
#elif args.recom_mode=='p_u':
# print(model.cal_pre_k(pred_dict, test_X))
pred_dict = model.cold_start_test(test_count1_X, user_feature_df,
item_feature_df)
print(model.cal_pre_k(pred_dict, test_count1_X))
elif FLAGS.mode == 'train':
# read data from file
train_file = 'data/train/train_u.csv'
train_X, train_y, user_pro_dict, pro_user_dict, user_index_map = utils.read_data(
train_file)
# read feature data from file
user_file = 'data/mid/user_features.csv'
item_file = 'data/mid/item_features.csv'
user_feature_df = utils.read_user_data(user_file, user_index_map)
item_feature_df = utils.read_item_data(item_file)
# generate batches
batch_loader = BatchLoader(args.batch_size, train_X, train_y)
args.num_users = np.max(train_X['uid']) + 1
args.num_items = len(item_feature_df) + 1
model = Model(args, args.recom_mode, batch_loader)
# train and save models
model.train(user_pro_dict, pro_user_dict, user_feature_df,
item_feature_df)
# save the user and index
utils.write_file(user_index_map['user'], "save/user_index_map")
elif FLAGS.mode == 'predict':
user_index_file = 'save/user_index_map'
user_index_map = pd.read_csv(user_index_file, names=['user'])
args.num_users = len(user_index_map)
user_file = 'data/data_pred/user_file.txt'
#user_file = 'data/mid/user_features.csv.liaoning'
user_df = pd.read_csv(user_file, names=['user'])
#user_df = pd.read_csv(user_file,header=None)
#user_df.rename(columns={0: 'user'}, inplace=True)
#user_df = user_df['user']
item_file = 'data/data_pred/item_file.txt'
item_df = pd.read_csv(item_file, names=['pid'])
user_index_map = user_index_map.append(
user_df[['user']]).drop_duplicates(['user']).reset_index(drop=True)
user_index_map['uid'] = user_index_map.index
user_df = pd.merge(user_df,
user_index_map,
left_on='user',
right_on='user',
how='left')
#del user_df['user']
# read feature data from file
user_file = 'data/mid/user_features.csv'
item_file = 'data/mid/item_features.csv'
user_feature_df = utils.read_user_data(user_file, user_index_map)
item_feature_df = utils.read_item_data(item_file)
args.num_items = len(item_feature_df) + 1
out_file = 'data/predict/predictions'
model = Model(args, args.recom_mode)
result_list = model.predict(user_df['uid'].values.tolist(),
item_df['pid'].values.tolist(),
user_feature_df, item_feature_df, out_file)
user_list = user_df['user'].values.tolist()
head_line = "cate_id,"
head_line += (',').join(user_list)
result_list.insert(0, head_line)
with codecs.open(out_file, 'w', encoding='utf-8') as f:
for val in result_list:
f.write(val + "\n")
else:
print('incorrect mode input...')
from batch_loader import BatchLoader
import config
import random
if __name__ == '__main__':
param = {"net": "pnet", "batch": 64}
batch_loader = BatchLoader(param)
task = random.choice(config.TRAIN_TASKS[param['net']])
data = batch_loader.next_batch(64, task)
for datum in data:
print(datum)
def main():
params = args()
model_prefix = params.model_prefix
load_epoch = params.load_epoch
batch_size = params.batch_size
img1_path = params.image1
img2_path = params.image2
img_shape = params.image_shape
img_dir = params.img_dir
txt_path = params.file_txt
# in out files
file_ = open(txt_path, 'r')
lines_ = file_.readlines()
result_ = open("result.txt", 'w')
#
if Test_Img == 'True':
img1 = cv2.imread(img1_path)
img2 = cv2.imread(img2_path)
img1 = cv2.resize(img1, (img_shape, img_shape))
img2 = cv2.resize(img2, (img_shape, img_shape))
img1 = np.expand_dims(img1, 0)
img2 = np.expand_dims(img2, 0)
test_batch_loader = BatchLoader("../data/facescrub_val.list", batch_size,
img_shape)
tf.reset_default_graph()
with tf.name_scope('input'):
input_images = tf.placeholder(tf.float32,
shape=(batch_size, img_shape, img_shape,
3),
name='input_images')
labels = tf.placeholder(tf.int32, shape=(batch_size), name='labels')
features, accuracy, pred_class, res1 = build_network(
input_images, labels, 526, 'test')
check_ckpt(model_prefix + '-' + str(load_epoch))
#detect
detector = face_dect()
#
with tf.Session() as sess:
restore_model = tf.train.Saver()
#restore_model = tf.train.import_meta_graph(model_prefix+'-'+str(load_epoch) +'.meta')
restore_model.restore(sess, model_prefix + '-' + str(load_epoch))
print("face model restore over")
'''
all_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
key_list = []
var_dic = dict()
for v_name in tf.global_variables():
i=0
print("name : ",v_name.name[:-2],v_name.shape)
print("shape",all_vars[i])
key_list.append(v_name.name[:-2])
i+=1
#print(tf.get_variable_scope())
#all_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
vas= sess.run([all_vars])
print(len(vas))
for i in range(len(vas)):
cur_name = key_list[i]
cur_var = vas[i]
print("name ,shape : ",cur_name,np.shape(cur_var))
var_dic[cur_name] = cur_var
'''
#restore_model = tf.train.import_meta_graph(model_prefix+'-'+str(load_epoch)+'.meta')
#restore_model.restore(sess,model_prefix+'-'+str(load_epoch))
if Test_Img == 'False':
iter_num = 0
accuracy_sum = 0
while iter_num < test_batch_loader.batch_num:
batch_images, batch_labels = test_batch_loader.next_batch()
images_in = (batch_images - 127.5) * 0.0078125
feat, batch_accuracy = sess.run([features, accuracy],
feed_dict={
input_images: images_in,
labels: batch_labels
})
accuracy_sum += batch_accuracy
iter_num += 1
if iter_num % 10 == 0:
print("step ", iter_num, batch_accuracy)
print("image num: ", test_batch_loader.data_num,
"the test accuracy: ",
accuracy_sum / (test_batch_loader.batch_num))
elif Test_Img == 'True':
with tf.name_scope('valdata'):
label_t = np.zeros([1])
feat1 = sess.run([features],
feed_dict={
input_images: img1,
labels: label_t
})
feat2 = sess.run([features],
feed_dict={
input_images: img2,
labels: label_t
})
distance = L2_distance(feat1, feat2, 512)
print("the 2 image dis: ", distance)
for rot, fdir, fname in os.walk(img_dir):
if len(fname) != 0:
break
img_list = []
print(fname)
for i in range(len(fname)):
org_img = cv2.imread(os.path.join(rot, fname[i]))
img_org = cv2.resize(org_img, (img_shape, img_shape))
img_org = np.expand_dims(img_org, 0)
img_list.append(img_org)
for i in range(len(fname)):
img1 = img_list[i]
feat1 = sess.run([features],
feed_dict={
input_images: img1,
labels: label_t
})
j = i + 1
while j < len(fname):
img2 = img_list[j]
t1 = time.time()
feat2 = sess.run([features],
feed_dict={
input_images: img2,
labels: label_t
})
t2 = time.time()
print("one image time ", t2 - t1)
distance = L2_distance(feat1, feat2, 512)
t3 = time.time()
print("compere time ", t3 - t2)
print(i, j, distance)
j += 1
elif Test_Img == 'File':
label_t = np.ones([1])
for i in range(len(lines_)):
feat_vec = []
feat1_fg = 0
feat2_fg = 0
line_1 = lines_[i]
line_1 = string.strip(line_1)
line_s = line_1.split(',')
dir_path_save = line_s[0][:-4]
dir_path_save = "../cropface/" + dir_path_save
mkdir_(dir_path_save)
for j in range(len(line_s)):
feat_vec2 = []
if j == 0:
#print("line ",line_s)
img1_pic = line_s[0]
img1_path = os.path.join(img_dir, img1_pic)
img1 = cv2.imread(img1_path)
bboxes_1 = detector.get_face(img1)
if bboxes_1 is not None:
for k in range(bboxes_1.shape[0]):
crop_img1 = get_img(img1, bboxes_1[k],
img_shape)
if k == 0 and SV_IMG:
img_save_path = dir_path_save + '/' + line_s[
0][:-4] + ".jpg"
save_image(img_save_path, crop_img1)
crop_img1 = (crop_img1 - 127.5) * 0.0078125
crop_img1 = np.expand_dims(crop_img1, 0)
feat1 = sess.run([features],
feed_dict={
input_images: crop_img1,
labels: label_t
})
print("a feature shape ", np.shape(feat1))
feat_vec.append(feat1)
feat_fg = 1
else:
print("a no face detect ")
break
else:
img2_pic = line_s[j]
img2_path = os.path.join(img_dir, img2_pic)
img2 = cv2.imread(img2_path)
bboxes_2 = detector.get_face(img2)
if bboxes_2 is not None:
for k in range(bboxes_2.shape[0]):
crop_img2 = get_img(img2, bboxes_2[k],
img_shape)
if SV_IMG:
img_save_path = dir_path_save + '/' + line_s[
j][:-4] + "-" + str(k) + ".jpg"
save_image(img_save_path, crop_img2)
crop_img2 = (crop_img2 - 127.5) * 0.0078125
crop_img2 = np.expand_dims(crop_img2, 0)
feat2 = sess.run([features],
feed_dict={
input_images: crop_img2,
labels: label_t
})
print("b feature shape ", np.shape(feat2))
feat_vec2.append(feat2)
feat2_fg = 1
else:
print("b no face detect ")
continue
if j > 0:
t2 = time.time()
distance = L2_distance(feat_vec[0], feat_vec2[0], 512)
print("distance is ", distance)
t3 = time.time()
print("compere time ", t3 - t2)
result_.write("{} ".format(distance))
result_.write("\n")
print(feat2)
file_.close()
result_.close()
class ContextEncoder_adv(object):
def __init__(self,
batch_size=params.BATCH_SIZE,
nb_epochs=params.NB_EPOCHS,
mask=None,
experiment_path=params.EXPERIMENT_PATH,
use_adversarial_loss=params.USE_ADVERSARIAL_LOSS,
lambda_decay=params.LAMBDA_DECAY,
lambda_adversarial=params.LAMBDA_ADVERSARIAL,
patience=params.PATIENCE,
discr_whole_image=params.DISCR_WHOLE_IMAGE,
discr_loss_limit=params.DISCR_LOSS_LIMIT,
use_dropout=params.USE_DROPOUT):
self.batch_size = batch_size
self.nb_epochs = nb_epochs
self.experiment_path = experiment_path
self.save_path = os.path.join(self.experiment_path, "model/")
self.save_best_path = os.path.join(self.experiment_path, "best_model/")
self.logs_path = os.path.join(self.experiment_path, "logs")
create_dir(self.save_path)
create_dir(self.logs_path)
self.global_step = tf.Variable(0, trainable=False, name='global_step')
self.phase = tf.placeholder(tf.bool, name='phase')
self.patience = patience
# parameters for adversarial loss
self.use_adversarial_loss = use_adversarial_loss
self.lambda_adversarial = lambda_adversarial
if lambda_decay:
self.lambda_adversarial = 1 - tf.train.exponential_decay(
.1, self.global_step, 10000, .5, staircase=True)
self.discr_whole_image = discr_whole_image
self.discr_loss_limit = discr_loss_limit
self.num_discr_trained = tf.Variable(tf.constant(0, dtype=tf.int32),
trainable=False)
self.use_dropout = use_dropout
self.batch_loader = BatchLoader(self.batch_size)
if mask is not None:
self.np_mask = mask
else:
self.np_mask = np.zeros((1, 64, 64, 1))
self.np_mask[:, 16:48, 16:48, :] = 1
self._sess = tf.Session()
tf.summary.scalar("lambda_adversarial", self.lambda_adversarial)
tf.summary.scalar("num discr trained", self.num_discr_trained)
def build_model(self):
# x : input
self.x = tf.placeholder(tf.float32, shape=[self.batch_size, 64, 64, 3])
# self.x_float = 2 * tf.image.convert_image_dtype(self.x, dtype=tf.float32) - 1
self.x_float = self.x / 255 * 2 - 1
self.mask = tf.placeholder(tf.float32, shape=[1, 64, 64, 1])
self.x_masked = self.x_float * (1 - self.mask)
self._encode()
self._channel_wise()
self._decode()
self._generate_image()
self._reconstruction_loss()
# adversarial loss
self._init_discriminator_variables()
self._adversarial_loss()
self._optimize()
self.merged_summary = tf.summary.merge_all()
def _encode(self):
with tf.name_scope("encode"):
with tf.name_scope('weights'):
self._W_conv1 = weight_variable([5, 5, 3, 128])
self._W_conv2 = weight_variable([5, 5, 128, 256])
self._W_conv3 = weight_variable([5, 5, 256, 512])
self._W_conv4 = weight_variable([5, 5, 512, 512])
self._W_conv5 = weight_variable([3, 3, 512, 512])
variable_summaries(self._W_conv1)
variable_summaries(self._W_conv2)
variable_summaries(self._W_conv3)
variable_summaries(self._W_conv4)
variable_summaries(self._W_conv5)
with tf.name_scope('biases'):
self._b_conv1 = bias_variable([128])
self._b_conv2 = bias_variable([256])
self._b_conv3 = bias_variable([512])
self._b_conv4 = bias_variable([512])
self._b_conv5 = bias_variable([512])
variable_summaries(self._b_conv1)
variable_summaries(self._b_conv2)
variable_summaries(self._b_conv3)
variable_summaries(self._b_conv4)
variable_summaries(self._b_conv5)
# 64 64 3
self.h_conv1 = activation_function(
conv2d(self.x_masked,
self._W_conv1,
stride=1,
is_training=self.phase) + self._b_conv1)
self.h_pool1 = avg_pool_2x2(self.h_conv1)
# 32 32 128
self.h_conv2 = activation_function(
conv2d(self.h_pool1,
self._W_conv2,
stride=1,
is_training=self.phase) + self._b_conv2)
self.h_pool2 = avg_pool_2x2(self.h_conv2)
# 16 16 256
self.h_conv3 = activation_function(
conv2d(self.h_pool2,
self._W_conv3,
stride=1,
is_training=self.phase) + self._b_conv3)
self.h_pool3 = avg_pool_2x2(self.h_conv3)
# 8 8 512
self.h_conv4 = activation_function(
conv2d(self.h_pool3,
self._W_conv4,
stride=1,
is_training=self.phase) + self._b_conv4)
self.h_pool4 = avg_pool_2x2(self.h_conv4)
# 4 4 512
self.h_conv5 = activation_function(
conv2d(self.h_pool4,
self._W_conv5,
stride=1,
is_training=self.phase) + self._b_conv5)
# 4 4 512
if self.use_dropout:
keep_prob = tf.cond(self.phase, lambda: tf.constant(.5),
lambda: tf.constant(1.))
self.h_conv5_drop = tf.nn.dropout(self.h_conv5, keep_prob)
else:
self.h_conv5_drop = self.h_conv5
def _channel_wise(self):
with tf.name_scope('channel_wise'):
with tf.name_scope('weights'):
self._W_fc1 = weight_variable([512, 4 * 4, 4 * 4])
variable_summaries(self._W_fc1)
with tf.name_scope('biases'):
self._b_fc1 = bias_variable([512])
variable_summaries(self._b_fc1)
self.h_conv5_flat_img = tf.reshape(self.h_conv5_drop,
[512, self.batch_size, 4 * 4])
self.h_fc1 = activation_function(
tf.reshape(tf.matmul(self.h_conv5_flat_img, self._W_fc1),
[self.batch_size, 16, 512]) + self._b_fc1)
self.h_fc1_img = tf.reshape(self.h_fc1,
[self.batch_size, 4, 4, 512])
def _decode(self):
with tf.name_scope('decode'):
with tf.name_scope('weights'):
self._W_uconv1 = weight_variable([5, 5, 512, 512])
self._W_uconv2 = weight_variable([5, 5, 256, 512])
self._W_uconv3 = weight_variable([5, 5, 128, 256])
with tf.name_scope('biases'):
self._b_uconv1 = bias_variable([512])
self._b_uconv2 = bias_variable([256])
self._b_uconv3 = bias_variable([128])
self.h_uconv1 = activation_function(
uconv2d(self.h_fc1_img,
self._W_uconv1,
output_shape=[self.batch_size, 8, 8, 512],
stride=2,
is_training=self.phase) + self._b_uconv1)
# 8 8 512
self.h_uconv2 = activation_function(
uconv2d(self.h_uconv1,
self._W_uconv2,
output_shape=[self.batch_size, 16, 16, 256],
stride=2,
is_training=self.phase) + self._b_uconv2)
# 16 16 256
self.h_uconv3 = activation_function(
uconv2d(self.h_uconv2,
self._W_uconv3,
output_shape=[self.batch_size, 32, 32, 128],
stride=2,
is_training=self.phase) + self._b_uconv3)
# 32 32 128
if self.use_dropout:
keep_prob = tf.cond(self.phase, lambda: tf.constant(.5),
lambda: tf.constant(1.))
self.h_uconv3_drop = tf.nn.dropout(self.h_uconv3, keep_prob)
else:
self.h_uconv3_drop = self.h_uconv3
def _generate_image(self):
with tf.name_scope('generated_image'):
self._W_uconv4 = weight_variable([5, 5, 3, 128])
self._b_uconv4 = bias_variable([3])
self.y = tf.nn.tanh(
uconv2d(self.h_uconv3_drop,
self._W_uconv4,
output_shape=[self.batch_size, 32, 32, 3],
stride=1,
is_training=self.phase) + self._b_uconv4)
# 32 32 3
self.y_padded = tf.pad(self.y,
[[0, 0], [16, 16], [16, 16], [0, 0]])
self.generated_image = self.y_padded + self.x_masked
# 64 64 3
tf.summary.image("original_image", self.x_float, max_outputs=12)
tf.summary.image("generated_image",
self.generated_image,
max_outputs=12)
def _reconstruction_loss(self):
with tf.name_scope('reconstruction_loss'):
self._reconstruction_loss = tf.nn.l2_loss(
self.mask * (self.x_float - self.y_padded)) / self.batch_size
tf.summary.scalar('reconstruction_loss', self._reconstruction_loss)
def _init_discriminator_variables(self):
with tf.name_scope('discriminator'):
with tf.name_scope('weights'):
self._W_discr1 = weight_variable([5, 5, 3, 128])
self._W_discr2 = weight_variable([5, 5, 128, 256])
self._W_discr3 = weight_variable([5, 5, 256, 512])
if self.discr_whole_image:
self._W_discr4 = weight_variable([5, 5, 512, 512])
self._W_dfc = weight_variable([4 * 4 * 512, 1])
with tf.name_scope('biases'):
self._b_discr1 = bias_variable([128])
self._b_discr2 = bias_variable([256])
self._b_discr3 = bias_variable([512])
if self.discr_whole_image:
self._b_discr4 = bias_variable([512])
self._b_dfc = bias_variable([1])
def _discriminator_encoder(self, image):
with tf.name_scope('discriminator_encoder'):
# image is 32 32 3 OR 64 64 3 (if whole image)
h_d1 = activation_function(
conv2d(image, self._W_discr1, stride=1, is_training=self.phase)
+ self._b_discr1)
h_dpool1 = avg_pool_2x2(h_d1)
# 16 16 128 OR 32 32 128 (if whole image)
h_d2 = activation_function(
conv2d(
h_dpool1, self._W_discr2, stride=1, is_training=self.phase)
+ self._b_discr2)
h_dpool2 = avg_pool_2x2(h_d2)
# 8 8 256 OR 16 16 256 (if whole image)
h_d3 = activation_function(
conv2d(
h_dpool2, self._W_discr3, stride=1, is_training=self.phase)
+ self._b_discr3)
h_dpool3 = avg_pool_2x2(h_d3)
if self.discr_whole_image:
# 8 8 512 (if whole image)
h_d4 = activation_function(
conv2d(h_dpool3,
self._W_discr4,
stride=1,
is_training=self.phase) + self._b_discr4)
h_dfinal = avg_pool_2x2(h_d4)
else:
h_dfinal = h_dpool3
# 4 4 512
h_dfinal_flat = tf.reshape(h_dfinal,
[self.batch_size, 4 * 4 * 512])
if self.use_dropout:
keep_prob = tf.cond(self.phase, lambda: tf.constant(.5),
lambda: tf.constant(1.))
h_dfinal_drop = tf.nn.dropout(h_dfinal_flat, keep_prob)
else:
h_dfinal_drop = h_dfinal_flat
discr = tf.matmul(h_dfinal_drop, self._W_dfc) + self._b_dfc
return discr
def _adversarial_loss(self):
with tf.name_scope('adversarial_loss'):
self._discr_variables = [
v for v in tf.trainable_variables()
if v.name.startswith('discriminator')
]
self._gen_variables = [
v for v in tf.trainable_variables()
if not v.name.startswith('discriminator')
]
print(len(self._discr_variables), "DISCR VARIABLES ",
[v.name for v in self._discr_variables])
print(len(self._gen_variables), "GEN VARIABLES",
[v.name for v in self._gen_variables])
if self.discr_whole_image:
# D(real img)
real_discr = self._discriminator_encoder(self.x_float)
# D(G(img))
fake_discr = self._discriminator_encoder(self.generated_image)
else:
# discriminate the center of the image only
self.real_img = tf.slice(self.x_float, [0, 16, 16, 0],
[self.batch_size, 32, 32, 3])
# D(real img)
real_discr = self._discriminator_encoder(self.real_img)
# D(G(img))
fake_discr = self._discriminator_encoder(self.y)
real_discr_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=real_discr, labels=.9 * tf.ones_like(real_discr)))
fake_discr_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=fake_discr, labels=tf.zeros_like(fake_discr)))
self._discr_adversarial_loss = (real_discr_loss +
fake_discr_loss) / 2
self._gen_adversarial_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=fake_discr, labels=tf.ones_like(fake_discr)))
# Disriminator loss
self._discr_loss = self._discr_adversarial_loss
# Generator loss (combination of reconstruction and adversarial loss)
self._gen_loss = self.lambda_adversarial * self._gen_adversarial_loss + \
(1 - self.lambda_adversarial) * self._reconstruction_loss
tf.summary.scalar("discr loss", self._discr_loss)
tf.summary.scalar("gen full loss (adversarial and reconstruction)",
self._gen_loss)
tf.summary.scalar("gen adversarial loss",
self._gen_adversarial_loss)
def _optimize(self):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.variable_scope("optimizer"):
optimizer = tf.train.AdamOptimizer()
with tf.control_dependencies(update_ops):
# Context-encoder
grads = optimizer.compute_gradients(self._reconstruction_loss)
self.train_fn = optimizer.apply_gradients(
grads, global_step=self.global_step)
# Context-encoder with adversarial loss
grads_discr = optimizer.compute_gradients(
loss=self._discr_loss, var_list=self._discr_variables)
grads_gen = optimizer.compute_gradients(
loss=self._gen_loss, var_list=self._gen_variables)
self.train_discr = optimizer.apply_gradients(
grads_discr, global_step=self.global_step)
self.train_gen = optimizer.apply_gradients(
grads_gen, global_step=self.global_step)
def _compute_val_loss(self):
n_val_batches = self.batch_loader.n_valid_batches // 2
val_loss = 0
for _ in tqdm(range(n_val_batches)):
batch = self.batch_loader.load_batch(train=False)
loss, summary_str = self._sess.run(
[self._reconstruction_loss, self.merged_summary],
feed_dict={
self.x: batch,
self.mask: self.np_mask,
self.phase: 0
})
val_loss += loss
val_loss /= n_val_batches
return val_loss, summary_str
def _restore(self):
"""
Retrieve last model saved if possible
Create a main Saver object
Create a SummaryWriter object
Init variables
:param save_name: string (default : model)
Name of the model
:return:
"""
saver = tf.train.Saver(max_to_keep=2)
# Try to restore an old model
last_saved_model = tf.train.latest_checkpoint(self.save_path)
self._sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(os.path.join(
self.logs_path, "train"),
graph=self._sess.graph,
flush_secs=20)
val_writer = tf.summary.FileWriter(os.path.join(self.logs_path, "val"),
graph=self._sess.graph,
flush_secs=20)
if last_saved_model is not None:
saver.restore(self._sess, last_saved_model)
print("[*] Restoring model {}".format(last_saved_model))
else:
tf.train.global_step(self._sess, self.global_step)
print("[*] New model created")
return saver, train_writer, val_writer
def train(self):
"""
Train the model
:return:
"""
# Retrieve a model or create a new
saver, train_writer, val_writer = self._restore()
epoch = 0
n_train_batches = self.batch_loader.n_train_batches
# Retrieve current global step
last_step = self._sess.run(self.global_step)
epoch += last_step // n_train_batches
last_iter = last_step - n_train_batches * epoch
print("last iter {}".format(last_iter))
print("last step {}".format(last_step))
print("epoch {}".format(epoch))
# Iterate over epochs
is_not_restart = False
patience_count = 0
best_val_loss = 1e10
while epoch < self.nb_epochs:
for i in tqdm(range(n_train_batches)):
if i < last_iter and not is_not_restart:
continue
is_not_restart = True
batch = self.batch_loader.load_batch(train=True)
if self.use_adversarial_loss:
# no discr_loss_limit
if self.discr_loss_limit >= 1:
_ = self._sess.run(self.train_discr,
feed_dict={
self.x: batch,
self.mask: self.np_mask,
self.phase: 1
})
# there is a discr_loss_limit
# train the discriminator only if its loss is higher than discr_loss_limit
else:
discr_loss = self._sess.run(self._discr_loss,
feed_dict={
self.x: batch,
self.mask:
self.np_mask,
self.phase: 1
})
if discr_loss >= self.discr_loss_limit:
self.num_discr_trained += 1
_ = self._sess.run(self.train_discr,
feed_dict={
self.x: batch,
self.mask: self.np_mask,
self.phase: 1
})
ops = [self.train_gen, self.global_step
] if self.use_adversarial_loss else [
self.train_fn, self.global_step
]
if i % 200 == 0:
ops.append(self.merged_summary)
output = self._sess.run(ops,
feed_dict={
self.x: batch,
self.mask: self.np_mask,
self.phase: 1
})
if i % 200 == 0:
# print("nb of black and white images so far : {}".format(self.nb_bw_img))
train_writer.add_summary(output[-1], global_step=output[1])
saver.save(self._sess,
global_step=output[1],
save_path=self.save_path)
val_loss, summary_str = self._compute_val_loss()
val_writer.add_summary(summary_str, global_step=output[1])
val_writer.add_summary(tf.Summary(value=[
tf.Summary.Value(tag="val_loss", simple_value=val_loss),
]),
global_step=output[1])
cprint("Epoch {}".format(epoch), color="yellow")
# early stopping
if val_loss < best_val_loss:
patience_count = 0
else:
patience_count += 1
if patience_count >= self.patience:
break
epoch += 1
cprint("Training done.", "green", attrs=["bold"])
train_writer.flush()
val_writer.flush()
train_writer.close()
val_writer.close()
def main():
LAMBDA = 1e-8
center_alpha = 0.9
num_class = 10000
embd_size = 512
args = argument()
checkpoint_dir = args.save_model_name
lr = args.lr
batch_size = args.batch_size
epoch_num = args.epoch_num
sta = args.sta
img_shape = args.img_size
train_file = args.train_file
val_file = args.val_file
train_batch_loader = BatchLoader(train_file, batch_size, img_shape)
test_batch_loader = BatchLoader(val_file, batch_size, img_shape)
#(Height,Width) = (train_batch_loader.height,train_batch_loader.width)
#train_batch_loader = mnist_data(batch_size)
print("img shape", img_shape)
with tf.name_scope('input'):
input_images = tf.placeholder(tf.float32,
shape=(batch_size, img_shape[0],
img_shape[1], 3),
name='input_images')
labels = tf.placeholder(tf.int32, shape=(batch_size), name='labels')
learn_rate = tf.placeholder(tf.float32,
shape=(None),
name='learn_rate')
with tf.name_scope('var'):
global_step = tf.Variable(0, trainable=False, name='global_step')
loss_op = CenterLoss(center_alpha, num_class, embd_size)
#with tf.device('/gpu:0'):
total_loss, accuracy, centers_update_op, center_loss, softmax_loss, pred_class, res1 = build_network(
input_images, labels, num_class, sta, loss_op, ratio=LAMBDA)
optimizer = tf.train.AdamOptimizer(learn_rate)
#optimizer = tf.train.GradientDescentOptimizer(learn_rate)
with tf.control_dependencies([centers_update_op]):
train_op = optimizer.minimize(total_loss, global_step=global_step)
#train_op = optimizer.minimize(total_loss, global_step=global_step)
summary_op = tf.summary.merge_all()
with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess:
#sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('../tmp/face_log', sess.graph)
saver = tf.train.Saver()
if args.pretrained is not None:
model_path = args.save_model_name + '-' + str(args.pretrained)
#saver.restore(sess,'./face_model/high_score-60')
saver.restore(sess, model_path)
else:
init = tf.global_variables_initializer()
sess.run(init)
step = sess.run(global_step)
epoch_idx = 0
graph_step = 0
while epoch_idx <= epoch_num:
step = 0
ckpt_fg = 'True'
ps_loss = 0.0
pc_loss = 0.0
acc_sum = 0.0
while step < train_batch_loader.batch_num:
batch_images, batch_labels = train_batch_loader.next_batch()
#batch_images, batch_labels = train_batch_loader.get_batchdata()
in_imgs = (batch_images - 127.5) * 0.0078125
#print("data in ",in_img[0,:2,:2,0])
_, summary_str, train_acc, Center_loss, Softmax_loss, Pred_class, res1_o = sess.run(
[
train_op, summary_op, accuracy, center_loss,
softmax_loss, pred_class, res1
],
feed_dict={
input_images: in_imgs,
labels: batch_labels,
learn_rate: lr
})
step += 1
#print("step",step, str(Softmax_loss),str(Center_loss))
#print("step label",step, str(batch_labels))
graph_step += 1
if step % 100 == 0:
writer.add_summary(summary_str, global_step=graph_step)
pc_loss += Center_loss
ps_loss += Softmax_loss
acc_sum += train_acc
if step % 1000 == 0:
#lr = lr*0.1
#c_loss+=c_loss
#s_loss+=s_loss
print("****** Epoch {} Step {}: ***********".format(
str(epoch_idx), str(step)))
print("center loss: {}".format(pc_loss / 1000.0))
print("softmax_loss: {}".format(ps_loss / 1000.0))
print("train_acc: {}".format(acc_sum / 1000.0))
print("centers", res1_o[0, :5])
print("*******************************")
#if (acc_sum/100.0) >= 0.98 and (pc_loss/100.0)<40 and (ps_loss/100.0) <0.1 and ckpt_fg=='True':
if ckpt_fg == 'True':
print(
"******************************************************************************"
)
saver.save(sess, checkpoint_dir, global_step=epoch_idx)
ckpt_fg = 'False'
ps_loss = 0.0
pc_loss = 0.0
acc_sum = 0.0
epoch_idx += 1
if epoch_idx % 10 == 0:
print(
"******************************************************************************"
)
saver.save(sess, checkpoint_dir, global_step=epoch_idx)
#writer.add_summary(summary_str, global_step=step)
if epoch_idx % 5 == 0:
lr = lr * 0.5
if epoch_idx:
batch_images, batch_labels = test_batch_loader.next_batch()
#batch_images,batch_labels = train_batch_loader.get_valdata()
vali_image = (batch_images - 127.5) * 0.0078125
vali_acc = sess.run(accuracy,
feed_dict={
input_images: vali_image,
labels: batch_labels
})
print(("epoch: {}, train_acc:{:.4f}, vali_acc:{:.4f}".format(
epoch_idx, train_acc, vali_acc)))
sess.close()
class TDNN(Model):
def __init__(self, sess, config):
self.sess = sess
self.config = config
self.tb_dir = "/data/tensorboard_log_dict/TDNN_white"
self.loader = BatchLoader(self.config)
def build_model(self, input_):
with tf.variable_scope("TDNN"):
with tf.variable_scope("conv") as scope:
maps = []
for idx, kernel_dim in enumerate(self.config.kernels):
#if idx < 3:
# gpu_num = 0
#else:
#gpu_num = idx-2
# gpu_num = 1
#with tf.device('/gpu:%d' % gpu_num):
reduced_length = input_.get_shape()[1] - kernel_dim + 1
# [batch_size x seq_length x embed_dim x feature_map_dim]
conv = layers.conv2d(
input_,
self.config.feature_maps[idx],
[kernel_dim, self.config.binary_embed_width],
1,
padding='VALID',
scope='conv' + str(idx),
weights_initializer=layers.xavier_initializer_conv2d())
# [batch_size x 1 x 1 x feature_map_dim]
pool = layers.max_pool2d(conv, [reduced_length, 1],
1,
scope='pool' + str(idx))
maps.append(tf.squeeze(pool))
fc = tf.concat(axis=1, values=maps)
with tf.variable_scope("fully"):
fc = tf.nn.dropout(fc, self.config.dropout_prob)
flat = tf.reshape(
fc,
[self.config.batch_size,
sum(self.config.feature_maps)])
fc1 = layers.fully_connected(
flat,
2048,
activation_fn=tf.nn.relu,
scope='fc1',
weights_initializer=layers.xavier_initializer(),
biases_initializer=tf.constant_initializer(0.01))
fc1 = tf.contrib.layers.batch_norm(fc1,
decay=0.9,
center=True,
scale=True,
epsilon=True,
activation_fn=tf.nn.relu)
fc2 = layers.fully_connected(
fc1,
1024,
activation_fn=tf.nn.relu,
scope='fc2',
weights_initializer=layers.xavier_initializer(),
biases_initializer=tf.constant_initializer(0.01))
fc2 = tf.contrib.layers.batch_norm(fc2,
decay=0.9,
center=True,
scale=True,
epsilon=True,
activation_fn=tf.nn.relu)
logits = layers.fully_connected(
fc2,
self.config.num_classes,
activation_fn=None,
scope='logits',
weights_initializer=layers.xavier_initializer(),
biases_initializer=tf.constant_initializer(0.01))
return logits
def build_loss(self, logits, labels):
weight_decay_rate = 0.0001
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy')
tf.add_to_collection(tf.GraphKeys.LOSSES, cross_entropy_mean)
tf.summary.scalar('loss_', cross_entropy_mean)
weights_only = filter(lambda x: x.name.endswith('w:0'),
tf.trainable_variables())
weight_decay = tf.reduce_sum(
tf.stack([tf.nn.l2_loss(x)
for x in weights_only])) * weight_decay_rate
cross_entropy_mean += weight_decay
return cross_entropy_mean
def inference(self, input_):
self.loader = BatchLoader(
self.config.data_dir, self.config.dataset_name,
self.config.batch_size, self.config.num_classes,
self.config.preprocessor, self.config.epoch,
self.config.specialist, self.config.forward_only)
content, filename = self.loader.prepare_inference()
#with tf.control_dependencies([self.loader.enqueue]):
logits = self.build_model(content)
softmax = tf.nn.softmax(logits)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
self.sess.run(init_op)
self.saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(self.config.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
self.saver.restore(self.sess, ckpt.model_checkpoint_path)
else:
print('no checkpoint found...')
self.sess.run(self.loader.enqueue,
feed_dict={self.loader.filenames: input_})
m_logits, m_softmax, m_filename = self.sess.run(
[logits, softmax, filename])
print(m_softmax, m_filename)
def tower_loss(self, scope, content, label, gpu_num):
#self.label = label
#self.path = path
logits = self.build_model(content)
loss = self.build_loss(logits, label)
self.accuracy = self.build_accuracy(logits, label)
tf.add_to_collection('losses', loss)
losses = tf.get_collection('losses', scope)
total_loss = tf.add_n(losses, name='total_loss')
for l in losses + [total_loss]:
loss_name = re.sub('%s_[0-9]*/' % 'tower', '', l.op.name)
return total_loss
def average_gradients(self, tower_grads):
average_grads = []
for grad_and_vars in zip(*tower_grads):
grads = []
for g, _ in grad_and_vars:
expanded_g = tf.expand_dims(g, 0)
grads.append(expanded_g)
grad = tf.concat(axis=0, values=grads)
grad = tf.reduce_mean(grad, 0)
v = grad_and_vars[0][1]
grad_and_var = (grad, v)
average_grads.append(grad_and_var)
return average_grads
def run(self):
self.global_step = tf.Variable(0, name="global_step", trainable=False)
if not self.config.forward_only:
self.opt = tf.train.AdamOptimizer(self.config.learning_rate)
tower_grads = []
content, label = self.loader.data_type_dict[self.config.data_type](
self.config.forward_only)
self.label = label
with tf.variable_scope(tf.get_variable_scope()):
for i in range(2):
with tf.device('/gpu:%d' % i):
with tf.name_scope('%s_%d' % ('tower', i)) as scope:
self.loss = self.tower_loss(
scope, content, label, i)
tf.get_variable_scope().reuse_variables()
grads = self.opt.compute_gradients(self.loss)
tower_grads.append(grads)
grads = self.average_gradients(tower_grads)
apply_gradient_op = self.opt.apply_gradients(
grads, global_step=self.global_step)
self.train_op = apply_gradient_op
else:
content, label = self.loader.data_type_dict[self.config.data_type](
self.config.forward_only)
self.label = label
logits = self.build_model(content)
self.accuracy = self.build_accuracy(logits, label)
# ready for train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
self.sess.run(init_op)
self.saver = tf.train.Saver()
self.merged_summary = tf.summary.merge_all()
self.writer = tf.summary.FileWriter(
"/data/tensorboard_log/dict/TDNN_0404/", self.sess.graph)
self.coord = tf.train.Coordinator()
self.threads = tf.train.start_queue_runners(self.sess, self.coord)
ckpt = tf.train.get_checkpoint_state(self.config.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
self.saver.restore(self.sess, ckpt.model_checkpoint_path)
else:
print('no checkpoint found...')
if not self.config.forward_only:
self.train()
#self.get_logits()
print("train")
else:
self.test()
def __init__(self, sess, config):
self.sess = sess
self.config = config
self.tb_dir = "/data/tensorboard_log_dict/TDNN_white"
self.loader = BatchLoader(self.config)
class LSTMTDNN(Model):
"""Time-delayed Nueral Network (cf. http://arxiv.org/abs/1508.06615v4)
"""
def __init__(self,
sess,
batch_size=100,
rnn_size=650,
layer_depth=2,
word_embed_dim=650,
char_embed_dim=15,
feature_maps=[50, 100, 150, 200, 200, 200, 200],
kernels=[1, 2, 3, 4, 5, 6, 7],
seq_length=35,
max_word_length=65,
use_word=False,
use_char=True,
hsm=0,
max_grad_norm=5,
highway_layers=2,
dropout_prob=0.5,
use_batch_norm=True,
checkpoint_dir="checkpoint",
forward_only=False,
data_dir="data",
dataset_name="pdb",
use_progressbar=False):
"""Initialize the parameters for LSTM TDNN
Args:
rnn_size: the dimensionality of hidden layers
layer_depth: # of depth in LSTM layers
batch_size: size of batch per epoch
word_embed_dim: the dimensionality of word embeddings
char_embed_dim: the dimensionality of character embeddings
feature_maps: list of feature maps (for each kernel width)
kernels: list of kernel widths
seq_length: max length of a word
use_word: whether to use word embeddings or not
use_char: whether to use character embeddings or not
highway_layers: # of highway layers to use
dropout_prob: the probability of dropout
use_batch_norm: whether to use batch normalization or not
hsm: whether to use hierarchical softmax
"""
self.sess = sess
self.batch_size = batch_size
self.seq_length = seq_length
# RNN
self.rnn_size = rnn_size
self.layer_depth = layer_depth
# CNN
self.use_word = use_word
self.use_char = use_char
self.word_embed_dim = word_embed_dim
self.char_embed_dim = char_embed_dim
self.feature_maps = feature_maps
self.kernels = kernels
# General
self.highway_layers = highway_layers
self.dropout_prob = dropout_prob
self.use_batch_norm = use_batch_norm
# Training
self.max_grad_norm = max_grad_norm
self.max_word_length = max_word_length
self.hsm = hsm
self.data_dir = data_dir
self.dataset_name = dataset_name
self.checkpoint_dir = checkpoint_dir
self.forward_only = forward_only
self.use_progressbar = use_progressbar
self.loader = BatchLoader(self.data_dir, self.dataset_name,
self.batch_size, self.seq_length,
self.max_word_length)
print('Word vocab size: %d, Char vocab size: %d, Max word length (incl. padding): %d' % \
(len(self.loader.idx2word), len(self.loader.idx2char), self.loader.max_word_length))
self.max_word_length = self.loader.max_word_length
self.char_vocab_size = len(self.loader.idx2char)
self.word_vocab_size = len(self.loader.idx2word)
# build LSTMTDNN model
self.prepare_model()
# load checkpoints
if self.forward_only == True:
if self.load(self.checkpoint_dir, self.dataset_name):
print(" [*] SUCCESS to load model for %s." % self.dataset_name)
else:
print(" [!] Failed to load model for %s." % self.dataset_name)
sys.exit(1)
def prepare_model(self):
with tf.variable_scope("LSTMTDNN"):
self.char_inputs = []
self.word_inputs = []
self.cnn_outputs = []
if self.use_char:
char_W = tf.get_variable(
"char_embed", [self.char_vocab_size, self.char_embed_dim])
else:
word_W = tf.get_variable(
"word_embed", [self.word_vocab_size, self.word_embed_dim])
with tf.variable_scope("CNN") as scope:
self.char_inputs = tf.placeholder(
tf.int32,
[self.batch_size, self.seq_length, self.max_word_length])
self.word_inputs = tf.placeholder(
tf.int32, [self.batch_size, self.seq_length])
char_indices = tf.split(1, self.seq_length, self.char_inputs)
word_indices = tf.split(1, self.seq_length,
tf.expand_dims(self.word_inputs, -1))
for idx in xrange(self.seq_length):
char_index = tf.reshape(char_indices[idx],
[-1, self.max_word_length])
word_index = tf.reshape(word_indices[idx], [-1, 1])
if idx != 0:
scope.reuse_variables()
if self.use_char:
# [batch_size x word_max_length, char_embed]
char_embed = tf.nn.embedding_lookup(char_W, char_index)
char_cnn = TDNN(char_embed, self.char_embed_dim,
self.feature_maps, self.kernels)
if self.use_word:
word_embed = tf.nn.embedding_lookup(
word_W, word_index)
cnn_output = tf.concat(1, char_cnn.output,
word_embed)
else:
cnn_output = char_cnn.output
else:
cnn_output = tf.squeeze(
tf.nn.embedding_lookup(word_W, word_index))
if self.use_batch_norm:
bn = batch_norm()
norm_output = bn(
tf.expand_dims(tf.expand_dims(cnn_output, 1), 1))
cnn_output = tf.squeeze(norm_output)
if highway:
#cnn_output = highway(input_, input_dim_length, self.highway_layers, 0)
cnn_output = highway(cnn_output,
cnn_output.get_shape()[1],
self.highway_layers, 0)
self.cnn_outputs.append(cnn_output)
with tf.variable_scope("LSTM") as scope:
self.cell = rnn_cell.BasicLSTMCell(self.rnn_size)
self.stacked_cell = rnn_cell.MultiRNNCell([self.cell] *
self.layer_depth)
outputs, _ = rnn.rnn(self.stacked_cell,
self.cnn_outputs,
dtype=tf.float32)
self.lstm_outputs = []
self.true_outputs = tf.placeholder(
tf.float32,
[self.batch_size, self.seq_length, self.word_vocab_size])
loss = 0
true_outputs = tf.split(1, self.seq_length, self.true_outputs)
for idx, (top_h,
true_output) in enumerate(zip(outputs,
true_outputs)):
if self.dropout_prob > 0:
top_h = tf.nn.dropout(top_h, self.dropout_prob)
if self.hsm > 0:
self.lstm_outputs.append(top_h)
else:
if idx != 0:
scope.reuse_variables()
proj = rnn_cell.linear(top_h, self.word_vocab_size, 0)
log_softmax = tf.log(tf.nn.softmax(proj))
self.lstm_outputs.append(log_softmax)
loss += tf.nn.softmax_cross_entropy_with_logits(
self.lstm_outputs[idx], tf.squeeze(true_output))
self.loss = tf.reduce_mean(loss) / self.seq_length
tf.scalar_summary("loss", self.loss)
tf.scalar_summary("perplexity", tf.exp(self.loss))
def train(self, epoch):
cost = 0
target = np.zeros(
[self.batch_size, self.seq_length, self.word_vocab_size])
N = self.loader.sizes[0]
for idx in xrange(N):
target.fill(0)
x, y, x_char = self.loader.next_batch(0)
for b in xrange(self.batch_size):
for t, w in enumerate(y[b]):
target[b][t][w] = 1
feed_dict = {
self.word_inputs: x,
self.char_inputs: x_char,
self.true_outputs: target,
}
_, loss, step, summary_str = self.sess.run(
[self.optim, self.loss, self.global_step, self.merged_summary],
feed_dict=feed_dict)
self.writer.add_summary(summary_str, step)
if idx % 50 == 0:
if self.use_progressbar:
progress(
idx / N, "epoch: [%2d] [%4d/%4d] loss: %2.6f" %
(epoch, idx, N, loss))
else:
print("epoch: [%2d] [%4d/%4d] loss: %2.6f" %
(epoch, idx, N, loss))
cost += loss
return cost / N
def test(self, split_idx, max_batches=None):
if split_idx == 1:
set_name = 'Valid'
else:
set_name = 'Test'
N = self.loader.sizes[split_idx]
if max_batches != None:
N = min(max_batches, N)
self.loader.reset_batch_pointer(split_idx)
target = np.zeros(
[self.batch_size, self.seq_length, self.word_vocab_size])
cost = 0
for idx in xrange(N):
target.fill(0)
x, y, x_char = self.loader.next_batch(split_idx)
for b in xrange(self.batch_size):
for t, w in enumerate(y[b]):
target[b][t][w] = 1
feed_dict = {
self.word_inputs: x,
self.char_inputs: x_char,
self.true_outputs: target,
}
loss = self.sess.run(self.loss, feed_dict=feed_dict)
if idx % 50 == 0:
if self.use_progressbar:
progress(
idx / N, "> %s: loss: %2.6f, perplexity: %2.6f" %
(set_name, loss, np.exp(loss)))
else:
print(" > %s: loss: %2.6f, perplexity: %2.6f" %
(set_name, loss, np.exp(loss)))
cost += loss
cost = cost / N
return cost
def run(self, epoch=25, learning_rate=1, learning_rate_decay=0.5):
self.current_lr = learning_rate
self.lr = tf.Variable(learning_rate, trainable=False)
self.opt = tf.train.GradientDescentOptimizer(self.lr)
#self.opt = tf.train.AdamOptimizer(learning_rate, beta1=0.5).minimize(self.loss)
# clip gradients
params = tf.trainable_variables()
grads = []
for grad in tf.gradients(self.loss, params):
if grad:
grads.append(tf.clip_by_norm(grad, self.max_grad_norm))
else:
grads.append(grad)
self.global_step = tf.Variable(0, name="global_step", trainable=False)
self.optim = self.opt.apply_gradients(zip(grads, params),
global_step=self.global_step)
# ready for train
tf.initialize_all_variables().run()
if self.load(self.checkpoint_dir, self.dataset_name):
print(" [*] SUCCESS to load model for %s." % self.dataset_name)
else:
print(" [!] Failed to load model for %s." % self.dataset_name)
self.saver = tf.train.Saver()
self.merged_summary = tf.merge_all_summaries()
self.writer = tf.train.SummaryWriter("./logs", self.sess.graph_def)
self.log_loss = []
self.log_perp = []
if not self.forward_only:
for idx in xrange(epoch):
train_loss = self.train(idx)
valid_loss = self.test(1)
# Logging
self.log_loss.append([train_loss, valid_loss])
self.log_perp.append([np.exp(train_loss), np.exp(valid_loss)])
state = {
'perplexity': np.exp(train_loss),
'epoch': idx,
'learning_rate': self.current_lr,
'valid_perplexity': np.exp(valid_loss)
}
print(state)
# Learning rate annealing
if len(self.log_loss) > 1 and self.log_loss[idx][
1] > self.log_loss[idx - 1][1] * 0.9999:
self.current_lr = self.current_lr * learning_rate_decay
self.lr.assign(self.current_lr).eval()
if self.current_lr < 1e-5: break
if idx % 2 == 0:
self.save(self.checkpoint_dir, self.dataset_name)
test_loss = self.test(2)
print(" [*] Test loss: %2.6f, perplexity: %2.6f" %
(test_loss, np.exp(test_loss)))
def main():
LAMBDA = 0.0
num_class = 526
checkpoint_dir = "../model/"
with tf.name_scope('input'):
input_images = tf.placeholder(tf.float32,
shape=(None, 100, 100, 3),
name='input_images')
labels = tf.placeholder(tf.int64, shape=(None), name='labels')
logits, features, total_loss, accuracy, centers_update_op, center_loss, softmax_loss = build_network(
input_images, labels, num_class, ratio=LAMBDA)
global_step = tf.Variable(0, trainable=False, name='global_step')
train_batch_loader = BatchLoader("../data/facescrub_train.list", 16)
test_batch_loader = BatchLoader("../data/facescrub_val.list", 16)
optimizer = tf.train.AdamOptimizer(0.001)
with tf.control_dependencies([centers_update_op]):
train_op = optimizer.minimize(total_loss, global_step=global_step)
summary_op = tf.summary.merge_all()
with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('../tmp/face_log', sess.graph)
saver = tf.train.Saver()
step = sess.run(global_step)
while step <= 80000:
batch_images, batch_labels = train_batch_loader.next_batch()
# print batch_images.shape
# print batch_labels.shape
_, summary_str, train_acc, Center_loss, Softmax_loss = sess.run(
[train_op, summary_op, accuracy, center_loss, softmax_loss],
feed_dict={
input_images:
(batch_images - 127.5) * 0.0078125, # - mean_data,
labels: batch_labels,
})
step += 1
print("step", step)
if step % 100 == 0:
print("********* Step %s: ***********" % str(step))
print("center loss: %s" % str(Center_loss))
print("softmax_loss: %s" % str(Softmax_loss))
print("train_acc: %s" % str(train_acc))
print("*******************************")
if step % 10000 == 0:
saver.save(sess,
checkpoint_dir + 'model.ckpt',
global_step=step)
writer.add_summary(summary_str, global_step=step)
if step % 2000 == 0:
batch_images, batch_labels = test_batch_loader.next_batch()
vali_image = (batch_images - 127.5) * 0.0078125
vali_acc = sess.run(accuracy,
feed_dict={
input_images: vali_image,
labels: batch_labels
})
print(("step: {}, train_acc:{:.4f}, vali_acc:{:.4f}".format(
step, train_acc, vali_acc)))
sess.close()
def __init__(self,
sess,
batch_size=100,
rnn_size=650,
layer_depth=2,
word_embed_dim=650,
char_embed_dim=15,
feature_maps=[50, 100, 150, 200, 200, 200, 200],
kernels=[1, 2, 3, 4, 5, 6, 7],
seq_length=35,
max_word_length=65,
use_word=False,
use_char=True,
hsm=0,
max_grad_norm=5,
highway_layers=2,
dropout_prob=0.5,
use_batch_norm=True,
checkpoint_dir="checkpoint",
forward_only=False,
data_dir="data",
dataset_name="pdb",
use_progressbar=False):
"""Initialize the parameters for LSTM TDNN
Args:
rnn_size: the dimensionality of hidden layers
layer_depth: # of depth in LSTM layers
batch_size: size of batch per epoch
word_embed_dim: the dimensionality of word embeddings
char_embed_dim: the dimensionality of character embeddings
feature_maps: list of feature maps (for each kernel width)
kernels: list of kernel widths
seq_length: max length of a word
use_word: whether to use word embeddings or not
use_char: whether to use character embeddings or not
highway_layers: # of highway layers to use
dropout_prob: the probability of dropout
use_batch_norm: whether to use batch normalization or not
hsm: whether to use hierarchical softmax
"""
self.sess = sess
self.batch_size = batch_size
self.seq_length = seq_length
# RNN
self.rnn_size = rnn_size
self.layer_depth = layer_depth
# CNN
self.use_word = use_word
self.use_char = use_char
self.word_embed_dim = word_embed_dim
self.char_embed_dim = char_embed_dim
self.feature_maps = feature_maps
self.kernels = kernels
# General
self.highway_layers = highway_layers
self.dropout_prob = dropout_prob
self.use_batch_norm = use_batch_norm
# Training
self.max_grad_norm = max_grad_norm
self.max_word_length = max_word_length
self.hsm = hsm
self.data_dir = data_dir
self.dataset_name = dataset_name
self.checkpoint_dir = checkpoint_dir
self.forward_only = forward_only
self.use_progressbar = use_progressbar
self.loader = BatchLoader(self.data_dir, self.dataset_name,
self.batch_size, self.seq_length,
self.max_word_length)
print('Word vocab size: %d, Char vocab size: %d, Max word length (incl. padding): %d' % \
(len(self.loader.idx2word), len(self.loader.idx2char), self.loader.max_word_length))
self.max_word_length = self.loader.max_word_length
self.char_vocab_size = len(self.loader.idx2char)
self.word_vocab_size = len(self.loader.idx2word)
# build LSTMTDNN model
self.prepare_model()
# load checkpoints
if self.forward_only == True:
if self.load(self.checkpoint_dir, self.dataset_name):
print(" [*] SUCCESS to load model for %s." % self.dataset_name)
else:
print(" [!] Failed to load model for %s." % self.dataset_name)
sys.exit(1)
if __name__ == '__main__':
# Use GPUs if available
args.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed for reproducible experiments
random.seed(args.seed)
torch.manual_seed(args.seed)
# Set the logger
utils.set_logger(os.path.join(args.model_dir, 'train.log'))
logging.info('device: {}'.format(args.device))
logging.info('Hyper params:%r' % args.__dict__)
# Create the input data pipeline
logging.info('Loading the datasets...')
bl = BatchLoader(args)
## Load train and dev data
train_data = bl.load_data('train.json')
dev_data = bl.load_data('dev.json')
## Train data
ner_train_data, re_train_data = bl.build_data(train_data, is_train=True)
train_bls = bl.batch_loader(ner_train_data,
re_train_data,
args.ner_max_len,
args.re_max_len,
args.batch_size,
is_train=True)
num_batchs_per_task = [len(train_bl) for train_bl in train_bls]
logging.info(
'num of batch per task for train: {}'.format(num_batchs_per_task))
train_task_ids = sum([[i] * num_batchs_per_task[i]