def train():
model_config = ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % FLAGS.model_type
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=3,
decay_rate=training_config.decay_factor)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up gradient clipping function
# def _clip_gradient_by_value(gvs):
# return [(tf.clip_by_value(grad, -training_config.clip_gradients,
# training_config.clip_gradients), var) for grad, var in gvs]
# grad_proc_fn = _clip_gradient_by_value
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver)
def train():
model_config = ModelConfig()
training_config = TrainConfig()
model_config.batch_size = 32
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.model_type, model_config.feat_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# create reader
reader = AttentionDataReader(batch_size=model_config.batch_size,
subset='trainval',
feat_type=model_config.feat_type)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict)
def train():
training_config = TrainConfig()
# Create training directory.
train_dir = FLAGS.train_dir % ('v1', 'Fusion')
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = RerankModel('train', version='v1', num_cands=5)
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
var_list = [
var for var in tf.global_variables() if 'Adam' not in var.name
]
saver = tf.train.Saver(
var_list, max_to_keep=training_config.max_checkpoints_to_keep)
# create reader
# reader = Reader(batch_size=32, subset='trainval', version='v1')
reader = Reader(batch_size=128 * 4, subset='trainval', version='v1')
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict,
debug_op=None)
def da_rnn(train_data: TrainData,
n_targs: int,
encoder_hidden_size=64,
decoder_hidden_size=64,
T=10,
learning_rate=0.01,
batch_size=128):
train_cfg = TrainConfig(T, int(train_data.feats.shape[0] * 0.7),
batch_size, nn.MSELoss())
logger.info(f"Training size:{train_cfg.train_size:d}.")
# 创建设置encoder参数设置
enc_kwargs = {
"input_size": train_data.feats.shape[1],
"hidden_size": encoder_hidden_size,
"T": T
}
encoder = Encoder(**enc_kwargs).to(device)
with open(os.path.join("data", "enc_kwargs.json"), "w") as fi:
json.dump(enc_kwargs, fi, indent=4) # Python数据结构转换为JSON,indent表示缩进
fi.close()
# 创建设置decoder参数设置
dec_kwargs = {
"encoder_hidden_size": encoder_hidden_size,
"decoder_hidden_size": decoder_hidden_size,
"T": T,
"out_feats": n_targs
}
decoder = Decoder(**dec_kwargs).to(device)
with open(os.path.join("data", "dec_kwargs.json"), "w") as fi:
json.dump(dec_kwargs, fi, indent=4)
fi.close()
# encoder、decoder优化器设置
encoder_optimizer = optim.Adam(
params=[p for p in encoder.parameters() if p.requires_grad],
lr=learning_rate)
decoder_optimizer = optim.Adam(
params=[p for p in decoder.parameters() if p.requires_grad],
lr=learning_rate)
da_rnn_net = DaRnnNet(encoder, decoder, encoder_optimizer,
decoder_optimizer)
return train_cfg, da_rnn_net
def __init__(self, init_model=None):
# params of the board and the game
self.game = Game()
# training params
self.config=TrainConfig()
self.data_buffer = deque(maxlen=self.config.buffer_size)
if init_model:
# start training from an initial policy-value net
self.policy_value_net = PolicyValueNet(init_model)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet()
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.config.c_puct,
n_playout=self.config.n_playout,
is_selfplay=1)
def black_box_function(lr_pow):
learning_rate = 10.0**lr_pow
results_dir = os.path.join(
savedir, "results" + '_{}'.format(modelname) +
'_{}'.format(args.outputformulation) + '_lr{}'.format(learning_rate) +
'_bs{}'.format(batch_size) + '_drop{}'.format(dropout))
print('writing results to {}'.format(results_dir))
dropouts = Dropouts(dropout, dropout, dropout)
doa_classes = None
if outputformulation == "Reg":
loss = nn.MSELoss(reduction='sum')
output_dimension = 3
elif outputformulation == "Class":
loss = nn.CrossEntropyLoss()
doa_classes = DoaClasses()
output_dimension = len(doa_classes.classes)
if modelname == "CNN":
model_choice = ConvNet(device, dropouts, output_dimension,
doa_classes).to(device)
elif modelname == "CRNN":
model_choice = CRNN(device, dropouts, output_dimension, doa_classes,
lstmout).to(device)
config = TrainConfig() \
.set_data_folder(inputdir) \
.set_learning_rate(learning_rate) \
.set_batch_size(batch_size) \
.set_num_epochs(epochs) \
.set_test_to_all_ratio(0.1) \
.set_results_dir(results_dir) \
.set_model(model_choice) \
.set_loss_criterion(loss) \
.set_doa_classes(doa_classes) \
.set_lstm_output(lstmout)
# negative sign for minimization
return -doa_train(config)
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
batchsize = 1
save_dir = train_conf.save_dir
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, L_filenames = read_sequential_target(L_data_dir, True)
print len(L_filenames)
B_fw, B_bw, B_bin, B_len, B_filenames = read_sequential_target(B_data_dir, True)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, L_filenames_u = read_sequential_target(L_data_dir_test, True)
print len(L_filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u, B_filenames_u = read_sequential_target(B_data_dir_test, True)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
L_enc_final_state = encoder(placeholders["L_bw"],
placeholders["L_len"],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_encoder")
##### sharing #####
L_shared = fc(L_enc_final_state, SHARE_DIM,
activation_fn=None, scope="L_share")
VB_dec_init_state = fc(L_shared, VB_num_units*VB_num_layers*2,
activation_fn=None, scope="VB_postshare")
##### decoding #####
B_output = VB_decoder(placeholders["V_fw"],
placeholders["B_fw"][0],
VB_dec_init_state,
length=B_shape[0],
out_dim=B_shape[2],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_decoder")
gpuConfig = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
for i in range(B_shape[1]):
feed_dict = {placeholders["L_fw"]: L_fw[:, i:i+1, :],
placeholders["B_fw"]: B_fw[:, i:i+1, :],
placeholders["V_fw"]: V_fw[:, i:i+1, :],
placeholders["L_bw"]: L_bw[:, i:i+1, :],
placeholders["B_bw"]: B_bw[:, i:i+1, :],
placeholders["V_bw"]: V_bw[:, i:i+1, :],
placeholders["B_bin"]: B_bin[:, i:i+1, :],
placeholders["L_len"]: L_len[i:i+1],
placeholders["V_len"]: V_len[i:i+1]}
result = sess.run(B_output, feed_dict=feed_dict)
save_latent(np.transpose(result, (1,0,2)), B_filenames[i], "generation")
if train_conf.test:
for i in range(B_shape_u[1]):
feed_dict = {placeholders["L_fw"]: L_fw_u[:, i:i+1, :],
placeholders["B_fw"]: B_fw_u[:, i:i+1, :],
placeholders["V_fw"]: V_fw_u[:, i:i+1, :],
placeholders["L_bw"]: L_bw_u[:, i:i+1, :],
placeholders["B_bw"]: B_bw_u[:, i:i+1, :],
placeholders["V_bw"]: V_bw_u[:, i:i+1, :],
placeholders["B_bin"]: B_bin_u[:, i:i+1, :],
placeholders["L_len"]: L_len_u[i:i+1],
placeholders["V_len"]: V_len_u[i:i+1]}
result = sess.run(B_output, feed_dict=feed_dict)
save_latent(np.transpose(result, (1,0,2)), B_filenames_u[i], "generation")
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
batchsize = train_conf.batchsize
epoch = train_conf.epoch
save_dir = train_conf.save_dir
if not os.path.exists(os.path.dirname(save_dir)):
os.mkdir(os.path.dirname(save_dir))
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, filenames = read_sequential_target(L_data_dir, True)
print len(filenames)
B_fw, B_bw, B_bin, B_len = read_sequential_target(B_data_dir)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, filenames_u = read_sequential_target(L_data_dir_test, True)
print len(filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u = read_sequential_target(B_data_dir_test)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
np.random.seed(seed)
tf.reset_default_graph()
tf.set_random_seed(seed)
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
L_enc_final_state = encoder(placeholders["L_bw"],
placeholders["L_len"],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_encoder")
VB_input = tf.concat([placeholders["V_bw"],
placeholders["B_bw"]],
axis=2)
VB_enc_final_state = encoder(VB_input,
placeholders["V_len"],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_encoder")
##### sharing #####
L_shared = fc(L_enc_final_state, SHARE_DIM,
activation_fn=None, scope="L_share")
VB_shared = fc(VB_enc_final_state, SHARE_DIM,
activation_fn=None, scope="VB_share")
L_dec_init_state = fc(L_shared, L_num_units*L_num_layers*2,
activation_fn=None, scope="L_postshare")
VB_dec_init_state = fc(VB_shared, VB_num_units*VB_num_layers*2,
activation_fn=None, scope="VB_postshare")
##### decoding #####
L_output = L_decoder(placeholders["L_fw"][0],
L_dec_init_state,
length=L_shape[0],
out_dim=L_shape[2],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_decoder")
B_output = VB_decoder(placeholders["V_fw"],
placeholders["B_fw"][0],
VB_dec_init_state,
length=B_shape[0],
out_dim=B_shape[2],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_decoder")
with tf.name_scope('loss'):
L_output = L_output # no need to multiply binary array
B_output = B_output * placeholders["B_bin"][1:]
L_loss = tf.reduce_mean(-tf.reduce_sum(placeholders["L_fw"][1:]*tf.log(L_output),
reduction_indices=[2]))
B_loss = tf.reduce_mean(tf.square(B_output-placeholders["B_fw"][1:]))
share_loss = aligned_discriminative_loss(L_shared, VB_shared)
loss = net_conf.L_weight*L_loss + net_conf.B_weight*B_loss + net_conf.S_weight*share_loss
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
gpuConfig = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
previous = time.time()
for step in range(epoch):
batch_idx = np.random.permutation(B_shape[1])[:batchsize]
feed_dict = {placeholders["L_fw"]: L_fw[:, batch_idx, :],
placeholders["B_fw"]: B_fw[:, batch_idx, :],
placeholders["V_fw"]: V_fw[:, batch_idx, :],
placeholders["L_bw"]: L_bw[:, batch_idx, :],
placeholders["B_bw"]: B_bw[:, batch_idx, :],
placeholders["V_bw"]: V_bw[:, batch_idx, :],
placeholders["B_bin"]: B_bin[:, batch_idx, :],
placeholders["L_len"]: L_len[batch_idx],
placeholders["V_len"]: V_len[batch_idx]}
_, l, b, s, t = sess.run([train_step,
L_loss,
B_loss,
share_loss,
loss],
feed_dict=feed_dict)
print "step:{} total:{}, language:{}, behavior:{}, share:{}".format(step, t, l, b, s)
if train_conf.test and (step+1) % train_conf.test_interval == 0:
batch_idx = np.random.permutation(B_shape_u[1])[:batchsize]
feed_dict = {placeholders["L_fw"]: L_fw_u[:, batch_idx, :],
placeholders["B_fw"]: B_fw_u[:, batch_idx, :],
placeholders["V_fw"]: V_fw_u[:, batch_idx, :],
placeholders["L_bw"]: L_bw_u[:, batch_idx, :],
placeholders["B_bw"]: B_bw_u[:, batch_idx, :],
placeholders["V_bw"]: V_bw_u[:, batch_idx, :],
placeholders["B_bin"]: B_bin_u[:, batch_idx, :],
placeholders["L_len"]: L_len_u[batch_idx],
placeholders["V_len"]: V_len_u[batch_idx]}
l, b, s, t = sess.run([L_loss, B_loss, share_loss, loss],
feed_dict=feed_dict)
print "test"
print "step:{} total:{}, language:{}, behavior:{}, share:{}".format(step, t, l, b, s)
if (step + 1) % train_conf.log_interval == 0:
saver.save(sess, save_dir)
past = time.time()
print past-previous
def train():
model_config = ModelConfig()
training_config = TrainConfig()
model_config.convert = FLAGS.convert
# model_config.batch_size = 2
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
reader_fn = create_reader(FLAGS.model_type, phase='train')
# setup environment
env = IVQARewards(metric='bleu')
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate
learning_rate = tf.constant(5e-5)
# def _learning_rate_decay_fn(learn_rate, global_step):
# return tf.train.exponential_decay(
# learn_rate,
# global_step,
# decay_steps=training_config.decay_step,
# decay_rate=training_config.decay_factor,
# staircase=False)
#
# learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=None,
variables=model.model_vars)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# create reader
reader = reader_fn(batch_size=16, subset='kptrain', version=FLAGS.version)
# Run training.
training_util.train(train_op,
model,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict,
env=env)
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# training_config.initial_learning_rate = 0.01
training_config.decay_step = 100000
training_config.decay_factor = 0.1
# training_config.optimizer = lambda: tf.train.MomentumOptimizer(0.5, momentum=0.5)
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
if FLAGS.sample_negative:
train_dir += '_sn'
if FLAGS.use_fb_data:
train_dir += '_fb'
if FLAGS.use_fb_bn:
train_dir += '_bn'
do_counter_sampling = FLAGS.version == 'v2'
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model_config.sample_negative = FLAGS.sample_negative
model_config.use_fb_bn = FLAGS.use_fb_bn
model = model_fn(model_config, phase='train')
model.build()
# Set up the learning rate
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=True)
# staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
# optimizer=tf.train.MomentumOptimizer(learning_rate, 0.9),
optimizer=training_config.optimizer,
clip_gradients=None,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# setup summaries
summary_op = tf.summary.merge_all()
# create reader
batch_size = 256 if FLAGS.sample_negative else 64
reader = Reader(batch_size=batch_size,
subset='kptrain',
sample_negative=FLAGS.sample_negative,
use_fb_data=FLAGS.use_fb_data)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict,
summary_op=summary_op)
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
batchsize = 1
epoch = train_conf.epoch
save_dir = train_conf.save_dir
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, L_filenames = read_sequential_target(
L_data_dir, True)
print len(L_filenames)
B_fw, B_bw, B_bin, B_len, B_filenames = read_sequential_target(
B_data_dir, True)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, L_filenames_u = read_sequential_target(
L_data_dir_test, True)
print len(L_filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u, B_filenames_u = read_sequential_target(
B_data_dir_test, True)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(
V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
tf.reset_default_graph()
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
L_enc_final_state = encoder(placeholders["L_bw"],
placeholders["L_len"],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_encoder")
VB_input = tf.concat([placeholders["V_bw"], placeholders["B_bw"]], axis=2)
VB_enc_final_state = encoder(VB_input,
placeholders["V_len"],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_encoder")
##### sharing #####
L_shared = fc(L_enc_final_state,
SHARE_DIM,
activation_fn=None,
scope="L_share")
VB_shared = fc(VB_enc_final_state,
SHARE_DIM,
activation_fn=None,
scope="VB_share")
gpuConfig = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
for i in range(B_shape[1]):
feed_dict = {
placeholders["L_fw"]: L_fw[:, i:i + 1, :],
placeholders["B_fw"]: B_fw[:, i:i + 1, :],
placeholders["V_fw"]: V_fw[:, i:i + 1, :],
placeholders["L_bw"]: L_bw[:, i:i + 1, :],
placeholders["B_bw"]: B_bw[:, i:i + 1, :],
placeholders["V_bw"]: V_bw[:, i:i + 1, :],
placeholders["B_bin"]: B_bin[:, i:i + 1, :],
placeholders["L_len"]: L_len[i:i + 1],
placeholders["V_len"]: V_len[i:i + 1]
}
L_enc, VB_enc = sess.run([L_shared, VB_shared], feed_dict=feed_dict)
save_latent(L_enc, L_filenames[i])
save_latent(VB_enc, B_filenames[i])
print B_filenames[i]
if train_conf.test:
for i in range(B_shape_u[1]):
feed_dict = {
placeholders["L_fw"]: L_fw_u[:, i:i + 1, :],
placeholders["B_fw"]: B_fw_u[:, i:i + 1, :],
placeholders["V_fw"]: V_fw_u[:, i:i + 1, :],
placeholders["L_bw"]: L_bw_u[:, i:i + 1, :],
placeholders["B_bw"]: B_bw_u[:, i:i + 1, :],
placeholders["V_bw"]: V_bw_u[:, i:i + 1, :],
placeholders["B_bin"]: B_bin_u[:, i:i + 1, :],
placeholders["L_len"]: L_len_u[i:i + 1],
placeholders["V_len"]: V_len_u[i:i + 1]
}
L_enc, VB_enc = sess.run([L_shared, VB_shared],
feed_dict=feed_dict)
save_latent(L_enc, L_filenames_u[i])
save_latent(VB_enc, B_filenames_u[i])
print B_filenames_u[i]
metavar="modeltype",
help="evaluation model type, coco or apollo",
default="apollo",
)
parser.add_argument(
"--submit",
metavar="/path/to/result/file",
help="path to result file",
default=ROOT_DIR+"/samples/apollo/submit/res_"+datetime.datetime.now().strftime("%Y%m%dT%H%M")+".csv",
)
args = parser.parse_args()
# config
if args.command == "train" or args.command == "trial":
config = TrainConfig()
if args.command == "trial":
config.STEPS_PER_EPOCH = 10
else:
config = TestConfig()
config.display()
# model
if args.command == "train" or args.command == "trial":
model = modellib.MaskRCNN(mode="training", config=config, model_dir=args.logs)
else:
model = modellib.MaskRCNN(mode="inference", config=config, model_dir=args.logs)
if args.model:
if args.model.lower() == "coco_backbone" or args.model.lower() == "coco":
model_path = COCO_MODEL_PATH
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.model_trainset, FLAGS.model_type)
do_counter_sampling = FLAGS.version == 'v2'
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, phase='train')
model.build()
# Set up the learning rate
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# setup summaries
summary_op = tf.summary.merge_all()
# create reader
# model_name = os.path.split(train_dir)[1]
reader = Reader(
batch_size=32,
subset=FLAGS.model_trainset,
cst_file='vqa_replay_buffer/vqa_replay_low_rescore_prior_05_04.json')
# reader = Reader(batch_size=64,
# known_set='kprestval',
# unknown_set='kptrain', # 'kptrain'
# un_ratio=1,
# hide_label=False)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict)
losses_train, 'train', tensorboard_writer, True)
@trainer.on(Events.EPOCH_COMPLETED)
def log_training_results(engine):
nonlocal best_loss
# evaluator.run(data_loader_train)
# losses_train = evaluator.state.metrics['loss']
evaluator.run(data_loader_val)
losses_val = evaluator.state.metrics['loss']
# log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer)
log_progress(trainer.state.epoch, trainer.state.iteration,
losses_val, 'val', tensorboard_writer)
if losses_val[exp.config.best_loss] < best_loss:
best_loss = losses_val[exp.config.best_loss]
save_weights(model, exp.experiment_dir.joinpath('best.th'))
tensorboard_dir = exp.experiment_dir.joinpath('log')
tensorboard_writer = SummaryWriter(str(tensorboard_dir))
trainer.run(data_loader_train, max_epochs=exp.config.num_epochs)
print(f'Experiment finished: {exp.experiment_id}')
if __name__ == '__main__':
main(TrainConfig())
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
batchsize = 1
epoch = train_conf.epoch
save_dir = train_conf.save_dir
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, L_filenames = read_sequential_target(L_data_dir, True)
print len(L_filenames)
B_fw, B_bw, B_bin, B_len, B_filenames = read_sequential_target(B_data_dir, True)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, L_filenames_u = read_sequential_target(L_data_dir_test, True)
print len(L_filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u, B_filenames_u = read_sequential_target(B_data_dir_test, True)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
tf.reset_default_graph()
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
L_enc_final_state = encoder(placeholders["L_bw"],
placeholders["L_len"],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_encoder")
VB_input = tf.concat([placeholders["V_bw"],
placeholders["B_bw"]],
axis=2)
VB_enc_final_state = encoder(VB_input,
placeholders["V_len"],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_encoder")
##### sharing #####
L_shared = fc(L_enc_final_state, SHARE_DIM,
activation_fn=None, scope="L_share")
VB_shared = fc(VB_enc_final_state, SHARE_DIM,
activation_fn=None, scope="VB_share")
gpuConfig = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
for i in range(B_shape[1]):
feed_dict = {placeholders["L_fw"]: L_fw[:, i:i+1, :],
placeholders["B_fw"]: B_fw[:, i:i+1, :],
placeholders["V_fw"]: V_fw[:, i:i+1, :],
placeholders["L_bw"]: L_bw[:, i:i+1, :],
placeholders["B_bw"]: B_bw[:, i:i+1, :],
placeholders["V_bw"]: V_bw[:, i:i+1, :],
placeholders["B_bin"]: B_bin[:, i:i+1, :],
placeholders["L_len"]: L_len[i:i+1],
placeholders["V_len"]: V_len[i:i+1]}
L_enc, VB_enc = sess.run([L_shared, VB_shared],
feed_dict=feed_dict)
save_latent(L_enc, L_filenames[i])
save_latent(VB_enc, B_filenames[i])
print B_filenames[i]
if train_conf.test:
for i in range(B_shape_u[1]):
feed_dict = {placeholders["L_fw"]: L_fw_u[:, i:i+1, :],
placeholders["B_fw"]: B_fw_u[:, i:i+1, :],
placeholders["V_fw"]: V_fw_u[:, i:i+1, :],
placeholders["L_bw"]: L_bw_u[:, i:i+1, :],
placeholders["B_bw"]: B_bw_u[:, i:i+1, :],
placeholders["V_bw"]: V_bw_u[:, i:i+1, :],
placeholders["B_bin"]: B_bin_u[:, i:i+1, :],
placeholders["L_len"]: L_len_u[i:i+1],
placeholders["V_len"]: V_len_u[i:i+1]}
L_enc, VB_enc = sess.run([L_shared, VB_shared],
feed_dict=feed_dict)
save_latent(L_enc, L_filenames_u[i])
save_latent(VB_enc, B_filenames_u[i])
print B_filenames_u[i]
def main():
encoder_conf = open("./encoder.json", "r")
encoder_layers = json.load(encoder_conf)
decoder_conf = open("./decoder.json", "r")
decoder_layers = json.load(decoder_conf)
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
q_units = net_conf.inference_num_units
q_layers = net_conf.inference_num_layers
p_units = net_conf.prediction_num_units
p_layers = net_conf.prediction_num_layers
latent_dim = net_conf.latent_dim
beta = net_conf.regularize_const
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
epoch = train_conf.epoch
save_dir = train_conf.save_dir + "/"
save_file = os.path.join(save_dir, "model.ckpt")
data_file = train_conf.data_file
if not os.path.exists(data_file):
if not os.path.exists(os.path.dirname(data_file)):
os.mkdir(os.path.dirname(data_file))
import wget
url = "http://www.cs.toronto.edu/~nitish/unsupervised_video/mnist_test_seq.npy"
wget.download(url, out=data_file)
data = np.load(data_file)
data = (data / 255.) * 0.9 + 0.05
train_data = data[:, :8000, :, :]
test_data = data[:, 8000:, :, :]
data_len = train_data.shape[0]
batchsize = train_conf.batchsize
height = train_data.shape[2]
width = train_data.shape[3]
n_channel = 1
np.random.seed(seed)
tf.compat.v1.reset_default_graph()
tf.compat.v1.set_random_seed(seed)
global_step = tf.Variable(0, name="global_step", trainable=False)
x = tf.compat.v1.placeholder(
tf.float32, [data_len, batchsize, height, width, n_channel])
reshaped_x = tf.reshape(x, [-1, height, width, n_channel])
with tf.compat.v1.variable_scope("encoder", reuse=False):
h = image_processor(reshaped_x, encoder_layers)
h = tf.reshape(h, [data_len, batchsize, -1])
z_post_mean, z_post_log_var = inference(h, q_units, q_layers, latent_dim)
z_prior_mean = tf.zeros(z_post_mean.get_shape().as_list(), tf.float32)
z_prior_log_var = tf.ones(z_post_log_var.get_shape().as_list(), tf.float32)
is_train = tf.compat.v1.placeholder(tf.bool)
z_mean = tf.cond(pred=is_train,
true_fn=lambda: z_post_mean,
false_fn=lambda: z_prior_mean)
z_log_var = tf.cond(pred=is_train,
true_fn=lambda: z_post_log_var,
false_fn=lambda: z_prior_log_var)
g = generation(h, z_mean, z_log_var, p_units, p_layers,
int(h.get_shape()[2]))
g = tf.reshape(g, [-1, int(h.get_shape()[2])])
with tf.compat.v1.variable_scope("decoder", reuse=False):
reshaped_y = image_processor(g, decoder_layers)
y = tf.reshape(reshaped_y, x.get_shape().as_list())
with tf.compat.v1.name_scope('loss'):
# mse is ok? not cross entropy?
recon_loss = tf.reduce_mean(input_tensor=tf.square(x - y))
latent_loss = -tf.reduce_mean(input_tensor=1 + z_log_var -
tf.square(z_mean) - tf.exp(z_log_var))
# what is the appropriate beta...
loss = recon_loss + beta * latent_loss
train_step = tf.compat.v1.train.AdamOptimizer(
learning_rate=0.001).minimize(loss, global_step=global_step)
gpuConfig = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.compat.v1.Session(config=gpuConfig)
sess.run(tf.compat.v1.global_variables_initializer())
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables())
ckpt = tf.train.get_checkpoint_state(save_dir)
latest_model = ckpt.model_checkpoint_path
saver.restore(sess, latest_model)
error_log = []
for itr in range(epoch):
batch_idx = np.random.permutation(8000)[:batchsize]
batch = train_data[:, batch_idx, :, :]
batch = np.reshape(batch, list(batch.shape) + [1])
feed_dict = {is_train: True, x: batch}
_, latent, recon, total, step = sess.run(
[train_step, latent_loss, recon_loss, loss, global_step],
feed_dict=feed_dict)
error_log.append([step, latent, recon, total])
print("step:{} latent:{}, recon:{}, total:{}".format(
step, latent, recon, total))
if train_conf.test and (step + 1) % train_conf.test_interval == 0:
batch_idx = np.random.permutation(2000)[:batchsize]
batch = test_data[:, batch_idx, :, :]
batch = np.reshape(batch, list(batch.shape) + [1])
feed_dict = {is_train: False, x: batch}
latent, recon, total = sess.run([latent_loss, recon_loss, loss],
feed_dict=feed_dict)
print("test.")
print("step:{} latent:{}, recon:{}, total:{}".format(
step, latent, recon, total))
if step % train_conf.log_interval == 0:
saver.save(sess, save_file, global_step=global_step)
error_log = np.array(error_log)
old_error_log = np.loadtxt("error.log")
np.savetxt("error.log", np.r_[old_error_log, error_log])
def multi_thread_op(i, samples, decodeImage, context, train_dataset,
with_mixup, mixupImage, photometricDistort, randomCrop,
randomFlipImage, normalizeBox, padBox, bboxXYXY2XYWH):
samples[i] = decodeImage(samples[i], context, train_dataset)
if with_mixup:
samples[i] = mixupImage(samples[i], context)
samples[i] = photometricDistort(samples[i], context)
samples[i] = randomCrop(samples[i], context)
samples[i] = randomFlipImage(samples[i], context)
samples[i] = normalizeBox(samples[i], context)
samples[i] = padBox(samples[i], context)
samples[i] = bboxXYXY2XYWH(samples[i], context)
if __name__ == '__main__':
cfg = TrainConfig()
class_names = get_classes(cfg.classes_path)
num_classes = len(class_names)
_anchors = copy.deepcopy(cfg.anchors)
num_anchors = len(cfg.anchor_masks[0]) # 每个输出层有几个先验框
_anchors = np.array(_anchors)
_anchors = np.reshape(_anchors, (-1, num_anchors, 2))
_anchors = _anchors.astype(np.float32)
# 步id，无需设置，会自动读。
iter_id = 0
# 多尺度训练
inputs = layers.Input(shape=(None, None, 3))
model_body = YOLOv4(inputs, num_classes, num_anchors)
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
batchsize = 1
save_dir = train_conf.save_dir
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, L_filenames = read_sequential_target(
L_data_dir, True)
print len(L_filenames)
B_fw, B_bw, B_bin, B_len, B_filenames = read_sequential_target(
B_data_dir, True)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, L_filenames_u = read_sequential_target(
L_data_dir_test, True)
print len(L_filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u, B_filenames_u = read_sequential_target(
B_data_dir_test, True)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(
V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
L_enc_final_state = encoder(placeholders["L_bw"],
placeholders["L_len"],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_encoder")
##### sharing #####
L_shared = fc(L_enc_final_state,
SHARE_DIM,
activation_fn=None,
scope="L_share")
VB_dec_init_state = fc(L_shared,
VB_num_units * VB_num_layers * 2,
activation_fn=None,
scope="VB_postshare")
##### decoding #####
B_output = VB_decoder(placeholders["V_fw"],
placeholders["B_fw"][0],
VB_dec_init_state,
length=B_shape[0],
out_dim=B_shape[2],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_decoder")
gpuConfig = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
for i in range(B_shape[1]):
feed_dict = {
placeholders["L_fw"]: L_fw[:, i:i + 1, :],
placeholders["B_fw"]: B_fw[:, i:i + 1, :],
placeholders["V_fw"]: V_fw[:, i:i + 1, :],
placeholders["L_bw"]: L_bw[:, i:i + 1, :],
placeholders["B_bw"]: B_bw[:, i:i + 1, :],
placeholders["V_bw"]: V_bw[:, i:i + 1, :],
placeholders["B_bin"]: B_bin[:, i:i + 1, :],
placeholders["L_len"]: L_len[i:i + 1],
placeholders["V_len"]: V_len[i:i + 1]
}
result = sess.run(B_output, feed_dict=feed_dict)
save_latent(np.transpose(result, (1, 0, 2)), B_filenames[i],
"generation")
if train_conf.test:
for i in range(B_shape_u[1]):
feed_dict = {
placeholders["L_fw"]: L_fw_u[:, i:i + 1, :],
placeholders["B_fw"]: B_fw_u[:, i:i + 1, :],
placeholders["V_fw"]: V_fw_u[:, i:i + 1, :],
placeholders["L_bw"]: L_bw_u[:, i:i + 1, :],
placeholders["B_bw"]: B_bw_u[:, i:i + 1, :],
placeholders["V_bw"]: V_bw_u[:, i:i + 1, :],
placeholders["B_bin"]: B_bin_u[:, i:i + 1, :],
placeholders["L_len"]: L_len_u[i:i + 1],
placeholders["V_len"]: V_len_u[i:i + 1]
}
result = sess.run(B_output, feed_dict=feed_dict)
save_latent(np.transpose(result, (1, 0, 2)), B_filenames_u[i],
"generation")
self.batch_count = raw_batch_count - raw_batch_count // self.config.validation_ratio
self.batch_count *= 2
def train(config, model_file, data_file):
model = load_model(model_file)
model.compile(
optimizer=SGD(lr=config.learning_rate, momentum=config.momentum),
loss=['mse', 'categorical_crossentropy'],
loss_weights=[1., 1.],
)
training_seq = GameData(config, data_file, False)
validation_seq = GameData(config, data_file, True)
model.fit_generator(
training_seq,
epochs=config.epochs,
steps_per_epoch=len(training_seq),
validation_data=validation_seq,
validation_steps=len(validation_seq),
callbacks=[
ModelCheckpoint(model_file, save_best_only=True),
EarlyStopping(patience=5),
],
)
if __name__ == '__main__':
train(TrainConfig(), sys.argv[1], sys.argv[2])
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
batchsize = train_conf.batchsize
epoch = train_conf.epoch
save_dir = train_conf.save_dir
if not os.path.exists(os.path.dirname(save_dir)):
os.mkdir(os.path.dirname(save_dir))
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, filenames = read_sequential_target(
L_data_dir, True)
print len(filenames)
B_fw, B_bw, B_bin, B_len = read_sequential_target(B_data_dir)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, filenames_u = read_sequential_target(
L_data_dir_test, True)
print len(filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u = read_sequential_target(
B_data_dir_test)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(
V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
np.random.seed(seed)
tf.reset_default_graph()
tf.set_random_seed(seed)
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
L_enc_final_state = encoder(placeholders["L_bw"],
placeholders["L_len"],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_encoder")
VB_input = tf.concat([placeholders["V_bw"], placeholders["B_bw"]], axis=2)
VB_enc_final_state = encoder(VB_input,
placeholders["V_len"],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_encoder")
##### sharing #####
L_shared = fc(L_enc_final_state,
SHARE_DIM,
activation_fn=None,
scope="L_share")
VB_shared = fc(VB_enc_final_state,
SHARE_DIM,
activation_fn=None,
scope="VB_share")
L_dec_init_state = fc(L_shared,
L_num_units * L_num_layers * 2,
activation_fn=None,
scope="L_postshare")
VB_dec_init_state = fc(VB_shared,
VB_num_units * VB_num_layers * 2,
activation_fn=None,
scope="VB_postshare")
##### decoding #####
L_output = L_decoder(placeholders["L_fw"][0],
L_dec_init_state,
length=L_shape[0],
out_dim=L_shape[2],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_decoder")
B_output = VB_decoder(placeholders["V_fw"],
placeholders["B_fw"][0],
VB_dec_init_state,
length=B_shape[0],
out_dim=B_shape[2],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_decoder")
with tf.name_scope('loss'):
L_output = L_output # no need to multiply binary array
B_output = B_output * placeholders["B_bin"][1:]
L_loss = tf.reduce_mean(-tf.reduce_sum(placeholders["L_fw"][1:] *
tf.log(L_output),
reduction_indices=[2]))
B_loss = tf.reduce_mean(tf.square(B_output - placeholders["B_fw"][1:]))
share_loss = aligned_discriminative_loss(L_shared, VB_shared)
loss = net_conf.L_weight * L_loss + net_conf.B_weight * B_loss + net_conf.S_weight * share_loss
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
gpuConfig = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
previous = time.time()
for step in range(epoch):
batch_idx = np.random.permutation(B_shape[1])[:batchsize]
feed_dict = {
placeholders["L_fw"]: L_fw[:, batch_idx, :],
placeholders["B_fw"]: B_fw[:, batch_idx, :],
placeholders["V_fw"]: V_fw[:, batch_idx, :],
placeholders["L_bw"]: L_bw[:, batch_idx, :],
placeholders["B_bw"]: B_bw[:, batch_idx, :],
placeholders["V_bw"]: V_bw[:, batch_idx, :],
placeholders["B_bin"]: B_bin[:, batch_idx, :],
placeholders["L_len"]: L_len[batch_idx],
placeholders["V_len"]: V_len[batch_idx]
}
_, l, b, s, t = sess.run(
[train_step, L_loss, B_loss, share_loss, loss],
feed_dict=feed_dict)
print "step:{} total:{}, language:{}, behavior:{}, share:{}".format(
step, t, l, b, s)
if train_conf.test and (step + 1) % train_conf.test_interval == 0:
batch_idx = np.random.permutation(B_shape_u[1])[:batchsize]
feed_dict = {
placeholders["L_fw"]: L_fw_u[:, batch_idx, :],
placeholders["B_fw"]: B_fw_u[:, batch_idx, :],
placeholders["V_fw"]: V_fw_u[:, batch_idx, :],
placeholders["L_bw"]: L_bw_u[:, batch_idx, :],
placeholders["B_bw"]: B_bw_u[:, batch_idx, :],
placeholders["V_bw"]: V_bw_u[:, batch_idx, :],
placeholders["B_bin"]: B_bin_u[:, batch_idx, :],
placeholders["L_len"]: L_len_u[batch_idx],
placeholders["V_len"]: V_len_u[batch_idx]
}
l, b, s, t = sess.run([L_loss, B_loss, share_loss, loss],
feed_dict=feed_dict)
print "test"
print "step:{} total:{}, language:{}, behavior:{}, share:{}".format(
step, t, l, b, s)
if (step + 1) % train_conf.log_interval == 0:
saver.save(sess, save_dir)
past = time.time()
print past - previous
logger.debug(model.get_model_class_balance_precision())
logger.info("mean accuracy: {}".format(
torch.mean(model.get_model_precision())))
logger.info("class_balance accuracy: {}".format(
torch.mean(model.get_model_class_balance_precision())))
logger.debug("validate mode end")
logger.info("--------------------------------------------------------")
if __name__ == '__main__':
# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# set_seed(2019)
config = TrainConfig()
config.continue_train = False # continue training: load the latest model
# config.continue_train = True # continue training: load the latest model
# config.load_iter = 16
logger = config.logger
config.batch_size = 80
logger.info("current batch size is {}".format(config.batch_size))
dataset = create_dataset(
config) # create a dataset given opt.dataset_mode and other options
model = create_model(config)
model.setup()
config.print_freq = int(len(dataset) / config.batch_size / 10)
total_iters = 0
logger.info("dataset len: %d " % len(dataset))
"results" + '_{}'.format(args.model) + '_{}'.format(args.outputformulation) + \
'_lr{}'.format(learning_rate) + '_bs{}'.format(batch_size) + '_drop{}'.format(args.dropout))
print('writing results to {}'.format(results_dir))
doa_classes = None
if args.outputformulation == "Reg":
loss = nn.MSELoss(reduction='sum')
output_dimension = 3
elif args.outputformulation == "Class":
loss = nn.CrossEntropyLoss(reduction="sum")
doa_classes = DoaClasses(doa_grid_resolution = np.pi/12)
output_dimension = len(doa_classes.classes)
if args.model == "CNN":
model_choice = ConvNet(device, dropouts, output_dimension, doa_classes).to(device)
elif args.model == "CRNN":
model_choice = CRNN(device, dropouts, output_dimension, doa_classes, args.lstmout).to(device)
config = TrainConfig() \
.set_data_folder(args.input) \
.set_learning_rate(learning_rate) \
.set_batch_size(batch_size) \
.set_num_epochs(args.epochs) \
.set_test_to_all_ratio(0.1) \
.set_results_dir(results_dir) \
.set_model(model_choice) \
.set_loss_criterion(loss) \
.set_doa_classes(doa_classes) \
.set_lstm_output(args.lstmout)
doa_train(config)
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
do_counter_sampling = FLAGS.version == 'v2'
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, phase='train')
model.build()
# Set up the learning rate
learning_rate = tf.constant(
training_config.initial_learning_rate) * 0.3
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn,
variables=model.model_vars)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# setup summaries
summary_op = tf.summary.merge_all()
# create reader
reader = Reader(
batch_size=64,
known_set='kprestval',
unknown_set='kptrain', # 'kptrain'
un_ratio=1,
hide_label=False)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
model=model,
summary_op=summary_op)
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
batchsize = 1
save_dir = train_conf.save_dir
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, L_filenames = read_sequential_target(
L_data_dir, True)
print len(L_filenames)
B_fw, B_bw, B_bin, B_len, B_filenames = read_sequential_target(
B_data_dir, True)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, L_filenames_u = read_sequential_target(
L_data_dir_test, True)
print len(L_filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u, B_filenames_u = read_sequential_target(
B_data_dir_test, True)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(
V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
tf.reset_default_graph()
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
VB_input = tf.concat([placeholders["V_bw"], placeholders["B_bw"]], axis=2)
VB_enc_final_state = encoder(VB_input,
placeholders["V_len"],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_encoder")
##### sharing #####
VB_shared = fc(VB_enc_final_state,
SHARE_DIM,
activation_fn=None,
scope="VB_share")
L_dec_init_state = fc(VB_shared,
L_num_units * L_num_layers * 2,
activation_fn=None,
scope="L_postshare")
##### decoding #####
L_output = L_decoder(placeholders["L_fw"][0],
L_dec_init_state,
length=L_shape[0],
out_dim=L_shape[2],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_decoder")
L_output = tf.contrib.seq2seq.hardmax(L_output)
gpuConfig = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
for i in range(B_shape[1]):
feed_dict = {
placeholders["L_fw"]: L_fw[:, i:i + 1, :],
placeholders["B_fw"]: B_fw[:, i:i + 1, :],
placeholders["V_fw"]: V_fw[:, i:i + 1, :],
placeholders["L_bw"]: L_bw[:, i:i + 1, :],
placeholders["B_bw"]: B_bw[:, i:i + 1, :],
placeholders["V_bw"]: V_bw[:, i:i + 1, :],
placeholders["B_bin"]: B_bin[:, i:i + 1, :],
placeholders["L_len"]: L_len[i:i + 1],
placeholders["V_len"]: V_len[i:i + 1]
}
result = sess.run([L_output], feed_dict=feed_dict)
r = result[0][:, 0, :].argmax(axis=1)
t = L_fw[1:, i, :].argmax(axis=1)
print(r == t).all()
if train_conf.test:
print "test!!!!!!!!!!!!!!!!"
for i in range(B_shape_u[1]):
feed_dict = {
placeholders["L_fw"]: L_fw_u[:, i:i + 1, :],
placeholders["B_fw"]: B_fw_u[:, i:i + 1, :],
placeholders["V_fw"]: V_fw_u[:, i:i + 1, :],
placeholders["L_bw"]: L_bw_u[:, i:i + 1, :],
placeholders["B_bw"]: B_bw_u[:, i:i + 1, :],
placeholders["V_bw"]: V_bw_u[:, i:i + 1, :],
placeholders["B_bin"]: B_bin_u[:, i:i + 1, :],
placeholders["L_len"]: L_len_u[i:i + 1],
placeholders["V_len"]: V_len_u[i:i + 1]
}
result = sess.run([L_output], feed_dict=feed_dict)
result = result[0][:, 0, :].argmax(axis=1)
target = L_fw_u[1:, i, :].argmax(axis=1)
print(result == target).all()
help="evaluation model type, coco or apollo",
default="apollo",
)
parser.add_argument(
"--submit",
metavar="/path/to/result/file",
help="path to result file",
default=ROOT_DIR + "/samples/apollo/submit/res_" +
datetime.datetime.now().strftime("%Y%m%dT%H%M") + ".csv",
)
args = parser.parse_args()
# config
if args.command == "train" or args.command == "trial":
config = TrainConfig()
if args.command == "trial":
config.STEPS_PER_EPOCH = 10
else:
config = TestConfig()
config.display()
# model
if args.command == "train" or args.command == "trial":
model = modellib.MaskRCNN(mode="training",
config=config,
model_dir=args.logs)
else:
model = modellib.MaskRCNN(mode="inference",
config=config,
model_dir=args.logs)
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
q_units = net_conf.inference_num_units
q_layers = net_conf.inference_num_layers
p_units = net_conf.prediction_num_units
p_layers = net_conf.prediction_num_layers
latent_dim = net_conf.latent_dim
beta = net_conf.regularize_const
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
epoch = train_conf.epoch
batch_len = train_conf.batch_length
batchsize = train_conf.batchsize
data_dim = 128
save_dir = train_conf.save_dir
if not os.path.exists(save_dir):
os.mkdir(save_dir)
save_file = os.path.join(save_dir, "model.ckpt")
# data preparation
train_data = read_target(train_conf.train_dir, data_dim)
train_num = train_data.shape[1]
train_len = train_data.shape[0]
if train_conf.test:
test_data = read_target(train_conf.test_dir)
test_len = test_data.shape[0]
test_num = test_data.shape[1]
np.random.seed(seed)
tf.reset_default_graph()
tf.set_random_seed(seed)
global_step = tf.Variable(0, name="global_step", trainable=False)
x = tf.placeholder(tf.float32, [batch_len, batchsize, data_dim])
z_mean, z_log_var = inference(x, q_units, q_layers, latent_dim)
y = generation(x, z_mean, z_log_var, p_units, p_layers, data_dim)
with tf.name_scope('loss'):
recon_loss = tf.reduce_mean(-tf.reduce_sum(x*tf.log(y), axis=2))
latent_loss = -tf.reduce_mean(tf.reduce_sum(1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var), axis=1))
loss = recon_loss + beta * latent_loss
train_step = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss, global_step=global_step)
gpuConfig = tf.ConfigProto(
gpu_options=tf.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(save_dir)
latest_model = ckpt.model_checkpoint_path
saver.restore(sess, latest_model)
error_log = []
for itr in range(epoch):
batch_idx = np.random.permutation(train_num)[:batchsize]
start_pos = np.random.randint(train_len - batch_len)
batch = train_data[start_pos:start_pos+batch_len, batch_idx, :]
feed_dict = {x: batch}
_, latent, recon, total, step = sess.run([train_step,
latent_loss,
recon_loss,
loss,
global_step],
feed_dict=feed_dict)
error_log.append([step, latent, recon, total])
print "step:{} latent:{}, recon:{}, total:{}".format(step, latent, recon, total)
if train_conf.test and itr % train_conf.test_interval == 0:
batch_idx = np.random.permutation(test_num)[:batchsize]
start_pos = np.random.randint(test_len - batch_len)
batch = test_data[start_pos:start_pos+batch_len, batch_idx, :]
feed_dict = {x: batch}
latent, recon, total = sess.run([latent_loss,
recon_loss,
loss],
feed_dict=feed_dict)
print "-------test--------"
print "step:{} latent:{}, recon:{}, total:{}".format(step, latent, recon, total)
print "-------------------"
if step % train_conf.log_interval == 0:
saver.save(sess, save_file, global_step=global_step)
error_log = np.array(error_log)
old_error_log = np.loadtxt("error.log")
np.savetxt("error.log", np.r_[old_error_log, error_log])
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
reader_fn = create_reader('VAQ-EpochAtt', phase='train')
env = MixReward(attention_vqa=True)
env.set_cider_state(use_cider=True)
env.diversity_reward.mode = 'winner_take_all'
env.set_language_thresh(1.0 / 3.0)
# env.set_replay_buffer(insert_thresh=0.1,
# sv_dir='vqa_replay_buffer/low_att') # if 0.5, already fooled others
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
ckpt_suffix = train_dir.split('/')[-1]
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.set_init_ckpt(
'model/v1_var_ivqa_restvalr2_VAQ-Var/model.ckpt-374000')
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate *
0.1)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Setup summaries
summary_op = tf.summary.merge_all()
# Setup language model
lm = LanguageModel()
lm.build()
lm.set_cache_dir(ckpt_suffix)
env.set_language_model(lm)
# create reader
reader = reader_fn(
batch_size=16,
subset='kprestval', # 'kptrain'
version=FLAGS.version)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
model=model,
summary_op=summary_op,
env=env)
tmp = tmp - X_train_mean
tmp = tmp / 255.0
yield tmp, y_train[idx]
def my_test_generator():
num_iters = X_test.shape[0] / batch_size
while 1:
for i in range(num_iters):
tmp = X_test[i*batch_size:(i+1)*batch_size].astype('float32')
tmp = tmp - X_train_mean
tmp = tmp / 255.0
yield tmp, y_test[i*batch_size:(i+1)*batch_size]
if __name__ == "__main__":
config = TrainConfig()
ch = config.num_channels
row = config.img_height
col = config.img_width
num_epoch = config.num_epoch
batch_size = config.batch_size
data_path = config.data_path
###[I]######################### INPUT OF THE MODEL#
#---------------------------------------------------
#loading trainng data
#---------------------------------------------------
print "Loading training data..."
def main():
encoder_conf = open("./encoder.json", "r")
encoder_layers = json.load(encoder_conf)
decoder_conf = open("./decoder.json", "r")
decoder_layers = json.load(decoder_conf)
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
q_units = net_conf.inference_num_units
q_layers = net_conf.inference_num_layers
p_units = net_conf.prediction_num_units
p_layers = net_conf.prediction_num_layers
latent_dim = net_conf.latent_dim
beta = net_conf.regularize_const
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
seed = train_conf.seed
epoch = train_conf.epoch
save_dir = train_conf.save_dir
if not os.path.exists(os.path.dirname(save_dir)):
os.mkdir(os.path.dirname(save_dir))
data_file = train_conf.data_file
if not os.path.exists(data_file):
if not os.path.exists(os.path.dirname(data_file)):
os.mkdir(os.path.dirname(data_file))
import wget
url = "http://www.cs.toronto.edu/~nitish/unsupervised_video/mnist_test_seq.npy"
wget.download(url, out=data_file)
data = np.load(data_file)
data = (data / 255.) * 0.9 + 0.05
train_data = data[:, :8000, :, :]
test_data = data[:, 8000:, :, :]
data_len = train_data.shape[0]
batchsize = 1
height = train_data.shape[2]
width = train_data.shape[3]
n_channel = 1
np.random.seed(seed)
tf.reset_default_graph()
tf.set_random_seed(seed)
x = tf.placeholder(tf.float32,
[data_len, batchsize, height, width, n_channel])
reshaped_x = tf.reshape(x, [-1, height, width, n_channel])
with tf.variable_scope("encoder", reuse=False):
h = image_processor(reshaped_x, encoder_layers)
h = tf.reshape(h, [data_len, batchsize, -1])
z_post_mean, z_post_log_var = inference(h, q_units, q_layers, latent_dim)
z_prior_mean = tf.zeros(z_post_mean.get_shape().as_list(), tf.float32)
z_prior_log_var = tf.ones(z_post_log_var.get_shape().as_list(), tf.float32)
is_train = tf.placeholder(tf.bool)
z_mean = tf.cond(is_train, lambda: z_post_mean, lambda: z_prior_mean)
z_log_var = tf.cond(is_train, lambda: z_post_log_var,
lambda: z_prior_log_var)
g = generation(h, z_mean, z_log_var, p_units, p_layers,
int(h.get_shape()[2]))
g = tf.reshape(g, [-1, int(h.get_shape()[2])])
with tf.variable_scope("decoder", reuse=False):
reshaped_y = image_processor(g, decoder_layers)
y = tf.reshape(reshaped_y, x.get_shape().as_list())
with tf.name_scope('loss'):
# mse is ok? not cross entropy?
recon_loss = tf.reduce_mean(tf.square(x - y))
latent_loss = -tf.reduce_mean(1 + z_log_var - tf.square(z_mean) -
tf.exp(z_log_var))
# what is the appropriate beta...
loss = recon_loss + beta * latent_loss
train_step = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
gpuConfig = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
error_log = []
for i in range(train_data.shape[1]):
print "train {}".format(i)
batch = train_data[:, i:i + 1, :, :]
batch = np.reshape(batch, list(batch.shape) + [1])
feed_dict = {is_train: True, x: batch}
outputs = sess.run(y, feed_dict=feed_dict)
save_as_images(outputs, i, "train")
for i in range(test_data.shape[1]):
print "test {}".format(i)
batch = test_data[:, i:i + 1, :, :]
batch = np.reshape(batch, list(batch.shape) + [1])
feed_dict = {is_train: True, x: batch}
outputs = sess.run(y, feed_dict=feed_dict)
save_as_images(outputs, i, "test")
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
do_counter_sampling = FLAGS.version == 'v2'
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, phase='train')
model.build()
# Set up the learning rate
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# setup summaries
summary_op = tf.summary.merge_all()
# create reader
model_name = os.path.split(train_dir)[1]
reader = Reader(batch_size=model_config.batch_size,
subset='trainval',
model_name=model_name,
epsilon=0.5,
feat_type='res5c',
version=FLAGS.version,
counter_sampling=do_counter_sampling)
# Run training.
train_framework_curriculum.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict,
loss_op=model.mle_losses,
summary_op=summary_op)
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Get model
reader_fn = create_reader('VAQ-Var', phase='train')
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
sample_fn = get_model_creation_fn('VAQ-VarRL')
sampler = sample_fn(model_config, 'train')
sampler.set_epsion(0.98)
sampler.build()
# Build language model
lm_fn = get_model_creation_fn(FLAGS.model_type)
language_model = lm_fn()
language_model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=language_model.loss,
learning_rate=learning_rate,
global_step=sampler.global_step,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
var_list = tf.get_collection(tf.GraphKeys.VARIABLES, 'LM')
saver = tf.train.Saver(
var_list=var_list,
max_to_keep=training_config.max_checkpoints_to_keep)
# Setup summaries
summary_op = tf.summary.merge_all()
# create reader
reader = reader_fn(
batch_size=16,
subset='kprestval', # 'kptrain'
version=FLAGS.version)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=sampler.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=sampler.init_fn,
saver=saver,
reader=reader,
model=language_model,
summary_op=summary_op,
sampler=sampler)
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
reader_fn = create_reader('VAQ-Var', phase='train')
env = MixReward()
env.diversity_reward.mode = 'winner_take_all'
env.set_cider_state(False)
env.set_language_thresh(0.2)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate *
0.1)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Setup summaries
summary_op = tf.summary.merge_all()
# Setup language model
lm = LanguageModel()
lm.build()
env.set_language_model(lm)
# create reader
reader = reader_fn(
batch_size=16,
subset='kprestval', # 'kptrain'
version=FLAGS.version)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
model=model,
summary_op=summary_op,
env=env)
def main():
net_conf = NetConfig()
net_conf.set_conf("./net_conf.txt")
L_num_units = net_conf.L_num_units
L_num_layers = net_conf.L_num_layers
VB_num_units = net_conf.VB_num_units
VB_num_layers = net_conf.VB_num_layers
SHARE_DIM = net_conf.S_dim
train_conf = TrainConfig()
train_conf.set_conf("./train_conf.txt")
batchsize = 1
save_dir = train_conf.save_dir
L_data_dir = train_conf.L_dir
B_data_dir = train_conf.B_dir
V_data_dir = train_conf.V_dir
L_fw, L_bw, L_bin, L_len, L_filenames = read_sequential_target(L_data_dir, True)
print len(L_filenames)
B_fw, B_bw, B_bin, B_len, B_filenames = read_sequential_target(B_data_dir, True)
V_fw, V_bw, V_bin, V_len = read_sequential_target(V_data_dir)
L_shape = L_fw.shape
B_shape = B_fw.shape
V_shape = V_fw.shape
if train_conf.test:
L_data_dir_test = train_conf.L_dir_test
B_data_dir_test = train_conf.B_dir_test
V_data_dir_test = train_conf.V_dir_test
L_fw_u, L_bw_u, L_bin_u, L_len_u, L_filenames_u = read_sequential_target(L_data_dir_test, True)
print len(L_filenames_u)
B_fw_u, B_bw_u, B_bin_u, B_len_u, B_filenames_u = read_sequential_target(B_data_dir_test, True)
V_fw_u, V_bw_u, V_bin_u, V_len_u = read_sequential_target(V_data_dir_test)
L_shape_u = L_fw_u.shape
B_shape_u = B_fw_u.shape
V_shape_u = V_fw_u.shape
tf.reset_default_graph()
placeholders = make_placeholders(L_shape, B_shape, V_shape, batchsize)
##### encoding #####
VB_input = tf.concat([placeholders["V_bw"],
placeholders["B_bw"]],
axis=2)
VB_enc_final_state = encoder(VB_input,
placeholders["V_len"],
n_units=VB_num_units,
n_layers=VB_num_layers,
scope="VB_encoder")
##### sharing #####
VB_shared = fc(VB_enc_final_state, SHARE_DIM,
activation_fn=None, scope="VB_share")
L_dec_init_state = fc(VB_shared, L_num_units*L_num_layers*2,
activation_fn=None, scope="L_postshare")
##### decoding #####
L_output = L_decoder(placeholders["L_fw"][0],
L_dec_init_state,
length=L_shape[0],
out_dim=L_shape[2],
n_units=L_num_units,
n_layers=L_num_layers,
scope="L_decoder")
L_output = tf.contrib.seq2seq.hardmax(L_output)
gpuConfig = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=train_conf.gpu_use_rate),
device_count={'GPU': 1})
sess = tf.Session(config=gpuConfig)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, save_dir)
for i in range(B_shape[1]):
feed_dict = {placeholders["L_fw"]: L_fw[:, i:i+1, :],
placeholders["B_fw"]: B_fw[:, i:i+1, :],
placeholders["V_fw"]: V_fw[:, i:i+1, :],
placeholders["L_bw"]: L_bw[:, i:i+1, :],
placeholders["B_bw"]: B_bw[:, i:i+1, :],
placeholders["V_bw"]: V_bw[:, i:i+1, :],
placeholders["B_bin"]: B_bin[:, i:i+1, :],
placeholders["L_len"]: L_len[i:i+1],
placeholders["V_len"]: V_len[i:i+1]}
result = sess.run([L_output], feed_dict=feed_dict)
r = result[0][:,0,:].argmax(axis=1)
t = L_fw[1:,i,:].argmax(axis=1)
print (r == t).all()
if train_conf.test:
print "test!!!!!!!!!!!!!!!!"
for i in range(B_shape_u[1]):
feed_dict = {placeholders["L_fw"]: L_fw_u[:, i:i+1, :],
placeholders["B_fw"]: B_fw_u[:, i:i+1, :],
placeholders["V_fw"]: V_fw_u[:, i:i+1, :],
placeholders["L_bw"]: L_bw_u[:, i:i+1, :],
placeholders["B_bw"]: B_bw_u[:, i:i+1, :],
placeholders["V_bw"]: V_bw_u[:, i:i+1, :],
placeholders["B_bin"]: B_bin_u[:, i:i+1, :],
placeholders["L_len"]: L_len_u[i:i+1],
placeholders["V_len"]: V_len_u[i:i+1]}
result = sess.run([L_output], feed_dict=feed_dict)
result = result[0][:,0,:].argmax(axis=1)
target = L_fw_u[1:,i,:].argmax(axis=1)
print (result == target).all()
