def main(_):
pp.pprint(flags.FLAGS.__flags)
cfg_stage_i = config_from_yaml(FLAGS.cfg_stage_I)
cfg = config_from_yaml(FLAGS.cfg_stage_II)
if not os.path.exists(cfg.CHECKPOINT_DIR):
os.makedirs(cfg.CHECKPOINT_DIR)
if not os.path.exists(cfg.SAMPLE_DIR):
os.makedirs(cfg.SAMPLE_DIR)
if not os.path.exists(cfg.LOGS_DIR):
os.makedirs(cfg.LOGS_DIR)
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
datadir = cfg.DATASET_DIR
dataset = TextDataset(datadir, 256)
filename_test = '%s/test' % datadir
dataset.test = dataset.get_data(filename_test)
filename_train = '%s/train' % datadir
dataset.train = dataset.get_data(filename_train)
with tf.Session(config=run_config) as sess:
if cfg.EVAL.FLAG:
stage_i = ConditionalGanStageI(cfg_stage_i, build_model=False)
stage_ii = ConditionalGan(stage_i, cfg, build_model=False)
stage_ii_eval = StageIIEval(
sess=sess,
model=stage_ii,
dataset=dataset,
cfg=cfg,
)
stage_ii_eval.evaluate_inception()
elif cfg.TRAIN.FLAG:
stage_i = ConditionalGanStageI(cfg_stage_i, build_model=False)
stage_ii = ConditionalGan(stage_i, cfg)
show_all_variables()
stage_ii_trainer = ConditionalGanTrainer(
sess=sess,
model=stage_ii,
dataset=dataset,
cfg=cfg,
cfg_stage_i=cfg_stage_i,
)
stage_ii_trainer.train()
else:
stage_i = ConditionalGanStageI(cfg_stage_i, build_model=False)
stage_ii = ConditionalGan(stage_i, cfg, build_model=False)
stage_ii_eval = StageIIVisualizer(
sess=sess,
model=stage_ii,
dataset=dataset,
cfg=cfg,
)
stage_ii_eval.visualize()
def main(_):
pp.pprint(flags.FLAGS.__flags)
cfg = config_from_yaml(FLAGS.cfg)
if not os.path.exists(cfg.CHECKPOINT_DIR):
os.makedirs(cfg.CHECKPOINT_DIR)
if not os.path.exists(cfg.SAMPLE_DIR):
os.makedirs(cfg.SAMPLE_DIR)
if not os.path.exists(cfg.LOGS_DIR):
os.makedirs(cfg.LOGS_DIR)
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
datadir = cfg.DATASET_DIR
dataset = TextDataset(datadir, 64)
# dataset = TextDataset(datadir, 512)
filename_test = '%s/test' % datadir
dataset._test = dataset.get_data(filename_test)
filename_train = '%s/train' % datadir
dataset.train = dataset.get_data(filename_train)
with tf.Session(config=run_config) as sess:
if cfg.EVAL.FLAG:
gancls = GanCls(cfg, build_model=False)
eval = GanClsEval(
sess=sess,
model=gancls,
dataset=dataset,
cfg=cfg)
eval.evaluate_inception()
elif cfg.TRAIN.FLAG:
gancls = GanCls(cfg)
show_all_variables()
gancls_trainer = GanClsTrainer(
sess=sess,
model=gancls,
dataset=dataset,
cfg=cfg,
)
gancls_trainer.train()
else:
gancls = GanCls(cfg, build_model=False)
gancls_visualiser = GanClsVisualizer(
sess=sess,
model=gancls,
dataset=dataset,
config=cfg,
)
gancls_visualiser.visualize()
def main(_):
if not FLAGS.output_file:
raise ValueError(
'You must supply the path to save to with --output_file')
# Read cmvn to do reverse mean variance normalization.
cmvn = np.load(os.path.join(FLAGS.data_dir, "train_cmvn.npz"))
with tf.Graph().as_default() as graph:
model = TfModel(rnn_cell=FLAGS.rnn_cell,
num_hidden=FLAGS.num_hidden,
dnn_depth=FLAGS.dnn_depth,
rnn_depth=FLAGS.rnn_depth,
output_size=FLAGS.output_dim,
bidirectional=FLAGS.bidirectional,
rnn_output=FLAGS.rnn_output,
cnn_output=FLAGS.cnn_output,
look_ahead=FLAGS.look_ahead,
mdn_output=FLAGS.mdn_output,
mix_num=FLAGS.mix_num,
name="tf_model")
input_sequence = tf.placeholder(name='input',
dtype=tf.float32,
shape=[None, FLAGS.input_dim])
length = tf.expand_dims(tf.shape(input_sequence)[0], 0)
# Apply normalization for input before inference.
mean_inputs = tf.constant(cmvn["mean_inputs"], dtype=tf.float32)
stddev_inputs = tf.constant(cmvn["stddev_inputs"], dtype=tf.float32)
input_sequence = (input_sequence - mean_inputs) / stddev_inputs
input_sequence = tf.expand_dims(input_sequence, 0)
output_sequence_logits, final_state = model(input_sequence, length)
# Apply reverse cmvn for output after inference
mean_labels = tf.constant(cmvn["mean_labels"], dtype=tf.float32)
stddev_labels = tf.constant(cmvn["stddev_labels"], dtype=tf.float32)
output_sequence_logits = output_sequence_logits * stddev_labels + mean_labels
output_sequence_logits = tf.squeeze(output_sequence_logits)
output_sequence_logits = tf.identity(output_sequence_logits,
name=FLAGS.output_node_name)
show_all_variables()
graph_def = graph.as_graph_def()
with gfile.GFile(FLAGS.output_file, 'wb') as f:
f.write(graph_def.SerializeToString())
#tf.train.write_graph(graph_def, './', 'inf_graph.pbtxt')
tf.logging.info("Inference graph has been written to %s" %
FLAGS.output_file)
def define_model(self):
self.x = tf.placeholder(tf.float32,
[self.cfg.TRAIN.BATCH_SIZE, 299, 299, 3],
name='inputs')
self.labels = tf.placeholder(tf.int32, [self.cfg.TRAIN.BATCH_SIZE])
self.logits, layers = inception_net(self.x,
self.cfg.MODEL.CLASSES,
for_training=True)
self.pred = tf.nn.softmax(self.logits)
train_correct_prediction = tf.equal(
self.labels, tf.cast(tf.argmax(self.pred, 1), tf.int32))
self.train_accuracy = tf.reduce_mean(
tf.cast(train_correct_prediction, tf.float32))
self.loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,
labels=self.labels))
self.vars_to_train = tf.trainable_variables('InceptionV3/Logits') \
+ tf.trainable_variables('InceptionV3/Mixed_7c')
self.all_vars = tf.global_variables('InceptionV3')
self.not_to_restore = tf.global_variables('InceptionV3/Logits') \
+ tf.global_variables('InceptionV3/AuxLogits')
self.pretrained_to_restore = [
var for var in self.all_vars if var not in self.not_to_restore
]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
opt = tf.train.RMSPropOptimizer(learning_rate=0.00005)
self.opt_step = opt.minimize(self.loss,
var_list=self.vars_to_train)
self.opt_vars = [
opt.get_slot(var, name) for name in opt.get_slot_names()
for var in self.vars_to_train
if opt.get_slot(var, name) is not None
]
show_all_variables()
def main(_):
pp.pprint(flags.FLAGS.__flags)
cfg = config_from_yaml(FLAGS.cfg)
if not os.path.exists(cfg.CHECKPOINT_DIR):
os.makedirs(cfg.CHECKPOINT_DIR)
if not os.path.exists(cfg.SAMPLE_DIR):
os.makedirs(cfg.SAMPLE_DIR)
if not os.path.exists(cfg.LOGS_DIR):
os.makedirs(cfg.LOGS_DIR)
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
datadir = cfg.DATASET_DIR
dataset = TextDataset(datadir, 1)
filename_test = '%s/test' % datadir
dataset._test = dataset.get_data(filename_test)
filename_train = '%s/train' % datadir
dataset.train = dataset.get_data(filename_train)
with tf.Session(config=run_config) as sess:
stage_i = ConditionalGan(cfg)
show_all_variables()
if cfg.TRAIN.FLAG:
stage_i_trainer = ConditionalGanTrainer(
sess=sess,
model=stage_i,
dataset=dataset,
cfg=cfg,
)
stage_i_trainer.train()
else:
pass
def build_model(self):
# Define the input tensor by appending the batch size dimension to the image dimension
self.iter = tf.placeholder(tf.int32, shape=None)
self.learning_rate = tf.placeholder(tf.float32, shape=None)
self.x = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, self.channel], name='x')
self.x_mismatch = tf.placeholder(tf.float32,
[self.batch_size, self.output_size, self.output_size, self.channel],
name='x_mismatch')
self.cond = tf.placeholder(tf.float32, [self.batch_size, self.embed_dim], name='cond')
self.z = tf.placeholder(tf.float32, [self.batch_size, self.z_dim], name='z')
self.epsilon = tf.placeholder(tf.float32, [self.batch_size, 1, 1, 1], name='eps')
self.z_sample = tf.placeholder(tf.float32, [self.sample_num] + [self.z_dim], name='z_sample')
self.cond_sample = tf.placeholder(tf.float32, [self.sample_num] + [self.embed_dim], name='cond_sample')
self.G, self.mean, self.log_sigma = self.generator(self.z, self.cond, stages=self.stage, t=self.trans)
self.Dg_logit = self.discriminator(self.G, self.cond, reuse=False, stages=self.stage, t=self.trans)
self.Dx_logit = self.discriminator(self.x, self.cond, reuse=True, stages=self.stage, t=self.trans)
self.Dxmi_logit = self.discriminator(self.x_mismatch, self.cond, reuse=True, stages=self.stage, t=self.trans)
self.epsilon = tf.random_uniform([self.batch_size, 1, 1, 1], 0., 1.)
self.x_hat = self.epsilon * self.G + (1. - self.epsilon) * self.x
self.cond_inp = self.cond + 0.0
self.Dx_hat_logit = self.discriminator(self.x_hat, self.cond_inp, reuse=True, stages=self.stage, t=self.trans)
self.sampler, _, _ = self.generator(self.z_sample, self.cond_sample, reuse=True, stages=self.stage,
t=self.trans)
self.alpha_assign = tf.assign(self.alpha_tra,
(tf.cast(tf.cast(self.iter, tf.float32) / self.steps, tf.float32)))
self.d_vars = tf.trainable_variables('d_net')
self.g_vars = tf.trainable_variables('g_net')
show_all_variables()
def main(_):
print("FLAG1")
pp.pprint(flags.FLAGS.__flags)
if FLAGS.input_width is None:
FLAGS.input_width = FLAGS.input_height
if FLAGS.output_width is None:
FLAGS.output_width = FLAGS.output_height
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
#gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
# extract zipfile
print(FLAGS.dataset)
print(os.path.join(FLAGS.data_path, "*.zip"))
source_path = glob.glob(os.path.join(FLAGS.data_path, "*.zip"))
print(source_path)
for i, zipped_file in enumerate(source_path):
print("Extracting image zip %s of %s" % (i + 1, len(source_path)))
if os.path.exists(os.path.join(FLAGS.data_path, "celebA")):
print("...File already exists")
else:
print(zipped_file)
unzip_and_save(zipped_file, FLAGS.data_path)
print("...Extracted!")
print("Reading from %s" % os.path.join(FLAGS.data_path, "*/*.jpg"))
unzipped_data_path = os.path.join(
FLAGS.data_path, "*/*.jpg") #right now we support only one dataset
print(unzipped_data_path)
with tf.Session(config=run_config) as sess:
if FLAGS.dataset == 'mnist':
dcgan = DCGAN(
sess,
input_width=FLAGS.input_width,
input_height=FLAGS.input_height,
output_width=FLAGS.output_width,
output_height=FLAGS.output_height,
batch_size=FLAGS.batch_size,
sample_num=FLAGS.batch_size,
y_dim=10,
data_path=FLAGS.data_path, #glob signature
dataset_type=unzipped_data_path,
crop=FLAGS.crop,
checkpoint_dir=FLAGS.checkpoint_dir,
sample_dir=FLAGS.sample_dir)
else:
dcgan = DCGAN(sess,
input_width=FLAGS.input_width,
input_height=FLAGS.input_height,
output_width=FLAGS.output_width,
output_height=FLAGS.output_height,
batch_size=FLAGS.batch_size,
sample_num=FLAGS.batch_size,
data_path=unzipped_data_path,
dataset_type=FLAGS.dataset,
crop=FLAGS.crop,
checkpoint_dir=FLAGS.checkpoint_dir,
sample_dir=FLAGS.sample_dir)
show_all_variables()
if FLAGS.train:
dcgan.train(FLAGS)
else:
if not dcgan.load(FLAGS.checkpoint_dir)[0]:
raise Exception("[!] Train a model first, then run test mode")
# to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0],
# [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1],
# [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2],
# [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3],
# [dcgan.h4_w, dcgan.h4_b, None])
# Below is codes for visualization
OPTION = 1
visualize(sess, dcgan, FLAGS, OPTION)
def main(_):
if not FLAGS.output_file:
raise ValueError(
'You must supply the path to save to with --output_file')
# Read cmvn to do reverse mean variance normalization.
cmvn = np.load(os.path.join(FLAGS.data_dir, "train_cmvn.npz"))
with tf.Graph().as_default() as graph:
model = TfModel(rnn_cell=FLAGS.rnn_cell,
dnn_depth=FLAGS.dnn_depth,
dnn_num_hidden=FLAGS.dnn_num_hidden,
rnn_depth=FLAGS.rnn_depth,
rnn_num_hidden=FLAGS.rnn_num_hidden,
output_size=FLAGS.output_dim,
bidirectional=FLAGS.bidirectional,
rnn_output=FLAGS.rnn_output,
cnn_output=FLAGS.cnn_output,
look_ahead=FLAGS.look_ahead,
mdn_output=FLAGS.mdn_output,
mix_num=FLAGS.mix_num,
name="tf_model")
input_sequence = tf.placeholder(name='input',
dtype=tf.float32,
shape=[None, FLAGS.input_dim])
length = tf.expand_dims(tf.shape(input_sequence)[0], 0)
# Apply normalization for input before inference.
mean_inputs = tf.constant(cmvn["mean_inputs"], dtype=tf.float32)
stddev_inputs = tf.constant(cmvn["stddev_inputs"], dtype=tf.float32)
input_sequence = (input_sequence - mean_inputs) / stddev_inputs
input_sequence = tf.expand_dims(input_sequence, 0)
output_sequence_logits, final_state = model(input_sequence, length)
# Apply reverse cmvn for output after inference
mean_labels = tf.constant(cmvn["mean_labels"], dtype=tf.float32)
stddev_labels = tf.constant(cmvn["stddev_labels"], dtype=tf.float32)
output_sequence_logits = output_sequence_logits * stddev_labels + mean_labels
output_sequence_logits = tf.squeeze(output_sequence_logits)
output_sequence_logits = tf.identity(output_sequence_logits,
name=FLAGS.output_node_name)
show_all_variables()
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path)
if ckpt:
saver = tf.train.Saver()
else:
tf.logging.warning("Cannot find checkpoint in {}".format(
args.checkpoint))
sys.exit(-1)
freeze_graph.freeze_graph_with_def_protos(
input_graph_def=graph.as_graph_def(),
input_saver_def=saver.as_saver_def(),
input_checkpoint=ckpt.model_checkpoint_path,
output_node_names=FLAGS.output_node_name,
restore_op_name=None,
filename_tensor_name=None,
output_graph=FLAGS.output_file,
clear_devices=True,
initializer_nodes="",
variable_names_blacklist=None)
tf.logging.info("Inference graph has been written to %s" %
FLAGS.output_file)
def train(cv_num_batch, tr_num_batch):
with tf.Graph().as_default():
with tf.device('/cpu:0'):
with tf.name_scope('input'):
tr_data_list = read_list(FLAGS.tr_list_file)
tr_inputs, tr_labels = get_batch(tr_data_list,
FLAGS.batch_size,
FLAGS.input_dim,
FLAGS.output_dim,
FLAGS.left_context,
FLAGS.right_context,
FLAGS.num_threads,
FLAGS.max_epoches)
cv_data_list = read_list(FLAGS.cv_list_file)
cv_inputs, cv_labels = get_batch(cv_data_list,
FLAGS.batch_size,
FLAGS.input_dim,
FLAGS.output_dim,
FLAGS.left_context,
FLAGS.right_context,
FLAGS.num_threads,
FLAGS.max_epoches)
devices = []
for i in xrange(FLAGS.num_gpu):
device_name = ("/gpu:%d" % i)
print('Using device: ', device_name)
devices.append(device_name)
# Prevent exhausting all the gpu memories.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# execute the session
with tf.Session(config=config) as sess:
# Create two models with tr_inputs and cv_inputs individually.
with tf.name_scope('model'):
print("=======================================================")
print(" Build Train model ")
print("=======================================================")
tr_model = SEGAN(sess, FLAGS, devices, tr_inputs, tr_labels,
cross_validation=False)
# tr_model and val_model should share variables
print("=======================================================")
print(" Build Cross-Validation model ")
print("=======================================================")
tf.get_variable_scope().reuse_variables()
cv_model = SEGAN(sess, FLAGS, devices, cv_inputs, cv_labels,
cross_validation=True)
show_all_variables()
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
print("Initializing variables ...")
sess.run(init)
if tr_model.load(tr_model.save_dir):
print("[*] Load SUCCESS")
else:
print("[!] Load failed, maybe begin a new model.")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
try:
cv_g_loss, cv_d_loss = eval_one_epoch(sess, coord, cv_model,
cv_num_batch)
print(("CROSSVAL PRERUN AVG.LOSS "
"%.4F(G_loss), %.4F(D_loss)") % (cv_g_loss, cv_d_loss))
sys.stdout.flush()
for epoch in xrange(FLAGS.max_epoches):
start = datetime.datetime.now()
tr_g_loss, tr_d_loss = train_one_epoch(sess, coord,
tr_model, tr_num_batch, epoch+1)
cv_g_loss, cv_d_loss = eval_one_epoch(sess, coord,
cv_model, cv_num_batch)
end = datetime.datetime.now()
print(("Epoch %02d: TRAIN AVG.LOSS "
"%.5F(G_loss, lrate(%e)), %.5F(D_loss, lrate(%e)), "
"CROSSVAL AVG.LOSS "
"%.5F(G_loss), %.5F(D_loss), TIME USED: %.2fmin") % (
epoch+1, tr_g_loss, tr_model.g_learning_rate,
tr_d_loss, tr_model.d_learning_rate,
cv_g_loss, cv_d_loss, (end-start).seconds/60.0))
sys.stdout.flush()
FLAGS.d_learning_rate *= FLAGS.halving_factor
FLAGS.g_learning_rate *= FLAGS.halving_factor
tr_model.d_learning_rate, tr_model.g_learning_rate = \
FLAGS.d_learning_rate, FLAGS.g_learning_rate
tr_model.save(tr_model.save_dir, epoch+1)
except Exception, e:
# Report exceptions to the coordinator.
coord.request_stop(e)
finally:
manual_annotation_file=None)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
model = ClassifierModel(sess,
epoch=num_epochs,
batch_size=_config.BATCH_SIZE,
z_dim=_config.Z_DIM,
dataset_name=DATASET_NAME,
beta=_config.beta,
num_units_in_layer=_config.num_units,
train_val_data_iterator=train_val_data_iterator,
log_dir=exp.config.LOG_PATH,
checkpoint_dir=exp.config.TRAINED_MODELS_PATH,
result_dir=exp.config.PREDICTION_RESULTS_PATH,
supervise_weight=exp.config.supervise_weight,
reconstruction_weight=exp.config.reconstruction_weight
)
exp.model = model
# show network architecture
show_all_variables()
exp.train(train_val_data_iterator)
train_val_data_iterator.reset_counter("train")
train_val_data_iterator.reset_counter("val")
exp.encode_latent_vector(train_val_data_iterator, num_epochs, "train")
train_val_data_iterator.reset_counter("train")
train_val_data_iterator.reset_counter("val")
exp.encode_latent_vector(train_val_data_iterator, num_epochs, "val")
def train():
"""Run the training of the acoustic or duration model."""
dataset_train = SequenceDataset(subset="train",
config_dir=FLAGS.config_dir,
data_dir=FLAGS.data_dir,
batch_size=FLAGS.batch_size,
input_size=FLAGS.input_dim,
output_size=FLAGS.output_dim,
num_threads=FLAGS.num_threads,
use_bucket=True,
infer=False,
name="dataset_train")()
dataset_valid = SequenceDataset(subset="valid",
config_dir=FLAGS.config_dir,
data_dir=FLAGS.data_dir,
batch_size=FLAGS.batch_size,
input_size=FLAGS.input_dim,
output_size=FLAGS.output_dim,
num_threads=FLAGS.num_threads,
use_bucket=True,
infer=False,
name="dataset_valid")()
model = TfModel(rnn_cell=FLAGS.rnn_cell,
dnn_depth=FLAGS.dnn_depth,
dnn_num_hidden=FLAGS.dnn_num_hidden,
rnn_depth=FLAGS.rnn_depth,
rnn_num_hidden=FLAGS.rnn_num_hidden,
output_size=FLAGS.output_dim,
bidirectional=FLAGS.bidirectional,
rnn_output=FLAGS.rnn_output,
cnn_output=FLAGS.cnn_output,
look_ahead=FLAGS.look_ahead,
mdn_output=FLAGS.mdn_output,
mix_num=FLAGS.mix_num,
name="tf_model")
# Build a reinitializable iterator for both dataset_train and dataset_valid.
iterator = tf.data.Iterator.from_structure(
dataset_train.batched_dataset.output_types,
dataset_train.batched_dataset.output_shapes)
(input_sequence, input_sequence_length, target_sequence,
target_sequence_length) = iterator.get_next()
training_init_op = iterator.make_initializer(dataset_train.batched_dataset)
validation_init_op = iterator.make_initializer(
dataset_valid.batched_dataset)
# Build the model and get the loss.
output_sequence_logits, train_final_state = model(input_sequence,
input_sequence_length)
loss = model.loss(output_sequence_logits, target_sequence,
target_sequence_length)
tf.summary.scalar("loss", loss)
learning_rate = tf.get_variable("learning_rate",
shape=[],
dtype=tf.float32,
initializer=tf.constant_initializer(
FLAGS.learning_rate),
trainable=False)
reduce_learning_rate = learning_rate.assign(
learning_rate * FLAGS.reduce_learning_rate_multiplier)
global_step = tf.get_variable(
name="global_step",
shape=[],
dtype=tf.int64,
initializer=tf.zeros_initializer(),
trainable=False,
collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP])
# Set up optimizer with global norm clipping.
trainable_variables = tf.trainable_variables()
optimizer = tf.train.AdamOptimizer(learning_rate)
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, trainable_variables),
FLAGS.max_grad_norm)
train_step = optimizer.apply_gradients(zip(grads, trainable_variables),
global_step=global_step)
show_all_variables()
merged_all = tf.summary.merge_all()
saver = tf.train.Saver(max_to_keep=FLAGS.max_epochs)
# Train
config = tf.ConfigProto()
# Prevent exhausting all the gpu memories
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Run init
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter(
os.path.join(FLAGS.save_dir, "nnet"), sess.graph)
if FLAGS.resume_training:
restore_from_ckpt(sess, saver)
sess.run(tf.assign(learning_rate, FLAGS.learning_rate))
# add a blank line for log readability
print()
sys.stdout.flush()
sess.run(validation_init_op)
loss_prev = eval_one_epoch(sess, loss, dataset_valid.num_batches)
tf.logging.info("CROSSVAL PRERUN AVG.LOSS %.4f\n" % loss_prev)
for epoch in range(FLAGS.max_epochs):
# Train one epoch
time_start = time.time()
sess.run(training_init_op)
tr_loss = train_one_epoch(sess, summary_writer, merged_all,
global_step, train_step, loss,
dataset_train.num_batches)
time_end = time.time()
used_time = time_end - time_start
# Validate one epoch
sess.run(validation_init_op)
val_loss = eval_one_epoch(sess, loss, dataset_valid.num_batches)
# Determine checkpoint path
FLAGS.learning_rate = sess.run(learning_rate)
cptk_name = 'nnet_epoch%d_lrate%g_tr%.4f_cv%.4f' % (
epoch + 1, FLAGS.learning_rate, tr_loss, val_loss)
checkpoint_path = os.path.join(FLAGS.save_dir, "nnet", cptk_name)
# accept or reject new parameters
if val_loss < loss_prev:
saver.save(sess, checkpoint_path)
# logging training loss along with validation loss
tf.logging.info("EPOCH %d: TRAIN AVG.LOSS %.4f, (lrate%g) "
"CROSSVAL AVG.LOSS %.4f, TIME USED %.2f, %s" %
(epoch + 1, tr_loss, FLAGS.learning_rate,
val_loss, used_time, "nnet accepted"))
loss_prev = val_loss
else:
tf.logging.info("EPOCH %d: TRAIN AVG.LOSS %.4f, (lrate%g) "
"CROSSVAL AVG.LOSS %.4f, TIME USED %.2f, %s" %
(epoch + 1, tr_loss, FLAGS.learning_rate,
val_loss, used_time, "nnet rejected"))
restore_from_ckpt(sess, saver)
# Reducing learning rate.
sess.run(reduce_learning_rate)
# add a blank line for log readability
print()
sys.stdout.flush()
def decode():
"""Run the decoding of the acoustic or duration model."""
with tf.device('/cpu:0'):
dataset_test = SequenceDataset(subset="test",
config_dir=FLAGS.config_dir,
data_dir=FLAGS.data_dir,
batch_size=1,
input_size=FLAGS.input_dim,
output_size=FLAGS.output_dim,
infer=True,
name="dataset_test")()
model = TfModel(rnn_cell=FLAGS.rnn_cell,
dnn_depth=FLAGS.dnn_depth,
dnn_num_hidden=FLAGS.dnn_num_hidden,
rnn_depth=FLAGS.rnn_depth,
rnn_num_hidden=FLAGS.rnn_num_hidden,
output_size=FLAGS.output_dim,
bidirectional=FLAGS.bidirectional,
rnn_output=FLAGS.rnn_output,
cnn_output=FLAGS.cnn_output,
look_ahead=FLAGS.look_ahead,
mdn_output=FLAGS.mdn_output,
mix_num=FLAGS.mix_num,
name="tf_model")
# Build the testing model and get test output sequence.
test_iterator = dataset_test.batched_dataset.make_one_shot_iterator()
input_sequence, input_sequence_length = test_iterator.get_next()
test_output_sequence_logits, test_final_state = model(
input_sequence, input_sequence_length)
show_all_variables()
saver = tf.train.Saver()
# Decode.
with tf.Session() as sess:
# Run init
sess.run(tf.global_variables_initializer())
if not restore_from_ckpt(sess, saver): sys.exit(-1)
# Read cmvn to do reverse mean variance normalization
cmvn = np.load(os.path.join(FLAGS.data_dir, "train_cmvn.npz"))
num_batches = 0
used_time_sum = frames_sum = 0.0
while True:
try:
time_start = time.time()
logits = sess.run(test_output_sequence_logits)
time_end = time.time()
used_time = time_end - time_start
used_time_sum += used_time
frame_num = logits.shape[1]
frames_sum += frame_num
# Squeeze batch dimension.
logits = logits.squeeze(axis=0)
if FLAGS.mdn_output:
out_pi = logits[:, :FLAGS.mix_num]
out_mu = logits[:, FLAGS.mix_num:(
FLAGS.mix_num + FLAGS.mix_num * FLAGS.output_dim)]
out_sigma = logits[:, (FLAGS.mix_num +
FLAGS.mix_num * FLAGS.output_dim):]
max_index_pi = out_pi.argmax(axis=1)
result_mu = []
for i in xrange(out_mu.shape[0]):
beg_index = max_index_pi[i] * FLAGS.output_dim
end_index = (max_index_pi[i] + 1) * FLAGS.output_dim
result_mu.append(out_mu[i, beg_index:end_index])
logits = np.vstack(result_mu)
sequence = logits * cmvn["stddev_labels"] + cmvn["mean_labels"]
out_dir_name = os.path.join(FLAGS.save_dir, "test", "cmp")
out_file_name = os.path.basename(
dataset_test.tfrecords_lst[num_batches]).split(
'.')[0] + ".cmp"
out_path = os.path.join(out_dir_name, out_file_name)
write_binary_file(sequence, out_path, with_dim=False)
#np.savetxt(out_path, sequence, fmt="%f")
tf.logging.info(
"writing inferred cmp to %s (%d frames in %.4f seconds)" %
(out_path, frame_num, used_time))
num_batches += 1
except tf.errors.OutOfRangeError:
break
tf.logging.info("Done decoding -- epoch limit reached (%d "
"frames per second)" % int(frames_sum / used_time_sum))
def main():
"""main"""
# parse arguments
args = parse_args()
if args is None:
exit()
# Slack Bot
if args.slack:
config = configparser.ConfigParser()
config.read('slack.config')
bot = SlackBot(token=config['SLACK']['token'],
channel_name=config['SLACK']['channel_name'])
else:
bot = None
gpu_options = tf.GPUOptions(visible_device_list=args.gpu_id,
allow_growth=True)
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1,
gpu_options=gpu_options,
allow_soft_placement=True)
# open session
models = [DCGAN, WGAN_GP, BEGAN, EBGAN]
with tf.Session(config=session_conf) as sess:
# declare instance for GAN
gan = None
for model in models:
if args.gan_type == model.model_name:
gan = model(sess,
epoch=args.epoch,
batch_size=args.batch_size,
z_dim=args.z_dim,
dataset_name=args.dataset,
compute_metrics_it=args.compute_metrics_it,
checkpoint_dir=args.checkpoint_dir,
result_dir=args.result_dir,
log_dir=args.log_dir,
gpu_id=args.gpu_id,
bot=bot,
redo=args.redo,
verbosity=args.verbosity)
if gan is None:
print("<!channel> ERROR!\n\n"
"[!] There is no option for " + args.gan_type)
raise Exception("[!] There is no option for " + args.gan_type)
# build graph
gan.build_model()
# show network architecture
show_all_variables()
# launch the graph in a session
gan.train()
print("[*] Training finished!")
# visualize learned generator
gan.visualize_results(args.epoch - 1)
print("[*] Testing finished!")
def train():
tr_tfrecords_lst, tr_num_batches = read_config_file(
"train_8k", FLAGS.batch_size)
val_tfrecords_lst, val_num_batches = read_config_file(
"cv_8k", FLAGS.batch_size)
with tf.Graph().as_default():
with tf.device('/cpu:0'):
with tf.name_scope('input'):
tr_inputs, tr_inputs_cmvn, tr_labels1, tr_labels2, tr_lengths = get_padded_batch(
tr_tfrecords_lst,
FLAGS.batch_size,
FLAGS.input_dim,
FLAGS.output_dim,
num_enqueuing_threads=FLAGS.num_threads,
num_epochs=FLAGS.max_epochs)
val_inputs, val_inputs_cmvn, val_labels1, val_labels2, val_lengths = get_padded_batch(
val_tfrecords_lst,
FLAGS.batch_size,
FLAGS.input_dim,
FLAGS.output_dim,
num_enqueuing_threads=FLAGS.num_threads,
num_epochs=FLAGS.max_epochs + 1)
# Create two models with train_input and val_input individually.
with tf.name_scope('model'):
tr_model = LSTM(FLAGS, tr_inputs_cmvn, tr_inputs, tr_labels1,
tr_labels2, tr_lengths)
# tr_model and val_model should share variables
tf.get_variable_scope().reuse_variables()
val_model = LSTM(FLAGS, val_inputs_cmvn, val_inputs, val_labels1,
val_labels2, val_lengths)
show_all_variables()
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# Prevent exhausting all the gpu memories.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.allow_soft_placement = True
sess = tf.Session(config=config)
sess.run(init)
if FLAGS.resume_training:
ckpt = tf.train.get_checkpoint_state(
os.path.join(FLAGS.save_dir + 'nnet'))
if ckpt and ckpt.model_checkpoint_path:
tf.logging.info("restore from" + ckpt.model_checkpoint_path)
tr_model.saver.restore(sess, ckpt.model_checkpoint_path)
else:
tf.logging.fatal("checkpoint not fo2und")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
# Cross validation before training.
loss_prev = eval_one_epoch(sess, coord, val_model, val_num_batches)
tf.logging.info("CROSSVAL PRERUN AVG.LOSS %.4F" % loss_prev)
sess.run(tf.assign(tr_model.lr, FLAGS.learning_rate))
for epoch in xrange(FLAGS.max_epochs):
start_time = time.time()
# Training
tr_loss = train_one_epoch(sess, coord, tr_model,
tr_num_batches)
# Validation
val_loss = eval_one_epoch(sess, coord, val_model,
val_num_batches)
end_time = time.time()
# Determine checkpoint path
ckpt_name = "nnet_iter%d_lrate%e_tr%.4f_cv%.4f" % (
epoch + 1, FLAGS.learning_rate, tr_loss, val_loss)
ckpt_dir = FLAGS.save_dir + '/nnet'
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
# Accept or reject new parameters
if val_loss < loss_prev:
tr_model.saver.save(sess, ckpt_path)
best_path = ckpt_path
# Logging train loss along with validation loss
tf.logging.info(
"ITERATION %d: TRAIN AVG.LOSS %.4f, (lrate%e) CROSSVAL"
" AVG.LOSS %.4f, %s (%s), TIME USED: %.2fs" %
(epoch + 1, tr_loss, FLAGS.learning_rate, val_loss,
"nnet accepted", ckpt_name,
(end_time - start_time) / 1))
else:
tr_model.saver.restore(sess, best_path)
tf.logging.info(
"ITERATION %d: TRAIN AVG.LOSS %.4f, (lrate%e) CROSSVAL"
" AVG.LOSS %.4f, %s, (%s), TIME USED: %.2fs" %
(epoch + 1, tr_loss, FLAGS.learning_rate, val_loss,
"nnet rejected", ckpt_name,
(end_time - start_time) / 1))
# Relative loss between previous and current val_loss
rel_impr = (loss_prev - val_loss) / loss_prev
loss_prev = val_loss
# Start halving when improvement is low
if rel_impr < FLAGS.start_halving_impr:
FLAGS.learning_rate *= FLAGS.halving_factor
sess.run(tf.assign(tr_model.lr, FLAGS.learning_rate))
# Stopping criterion
if rel_impr < FLAGS.end_halving_impr:
if epoch < FLAGS.min_epochs:
tf.logging.info(
"we were supposed to finish, but we continue as "
"min_epochs : %s" % FLAGS.min_epochs)
continue
else:
tf.logging.info(
"finished, too small rel. improvement %g" %
rel_impr)
break
except Exception, e:
# Report exceptions to the coordinator.
coord.request_stop(e)
finally:
