def visualize(conf, refeed_img = True):
image_batch, actions, states = build_tfrecord_input(conf, training=True)
with tf.variable_scope('model'):
if refeed_img:
model = Model(conf, test=True)
else:
encoder = Encoder(conf)
pred_and_dec = Predictor_and_Decoder(conf)
print 'Constructing saver.'
# Make saver.
saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.VARIABLES), max_to_keep=0)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
# Make training session.
sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))
tf.train.start_queue_runners(sess)
sess.run(tf.initialize_all_variables())
saver.restore(sess, conf['visualize'])
image_batch_raw, actions, states = sess.run([image_batch, actions, states])
states = np.split(states, 1)
image_batch = np.split(image_batch_raw, conf['sequence_length'], axis=1)
image_batch = [np.squeeze(img) for img in image_batch]
gen_images = [np.zeros([conf['batch_size'], 64, 64, 3]) for _ in range(conf['sequence_length'])]
gen_images[0] = image_batch[0]
gen_images[1] = image_batch[1]
lt_states = []
# refeeding images
for t in range(1,conf['sequence_length']-1):
gen_img0 = np.expand_dims(deepcopy(gen_images[t-1]), axis= 1)
gen_img1 = np.expand_dims(deepcopy(gen_images[t]), axis=1)
images01 = np.concatenate((gen_img0,gen_img1), axis=1)
if refeed_img:
feed_dict ={
model.images_01: images01,
model.actions_01: actions[:, t-1: t+1],
}
[images_12] = sess.run([model.images_12_rec],
feed_dict)
else: # propagate latent..
if t==1:
#encode
feed_dict = {
encoder.images_01: images01,
}
[inf_lt_state01] = sess.run([encoder.inf_lt_state01], feed_dict)
lt_states.append(inf_lt_state01)
# predict and decode:
feed_dict = {
pred_and_dec.lt_state_01: lt_states[-1],
pred_and_dec.actions_01: actions[:, t-1: t+1]
}
[pred_lt_state_23, images_12] = sess.run([pred_and_dec.pred_lt_state_23 , pred_and_dec.images_rec],
feed_dict)
lt_states.append(pred_lt_state_23)
images_2 = images_12[:, :, :, 3:6] # 0:3
gen_images[t + 1] = images_2
file_path = conf['output_dir']
cPickle.dump(gen_images, open(file_path + '/gen_image_seq.pkl', 'wb'))
cPickle.dump(image_batch_raw, open(file_path + '/ground_truth.pkl', 'wb'))
print 'written files to:' + file_path
video_prediction.utils_vpred.create_gif.comp_video(conf['output_dir'], conf, suffix='_refimg{}'.format(refeed_img))
sess.close()
def main(unused_argv):
os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS.device)
print 'using CUDA_VISIBLE_DEVICES=', FLAGS.device
from tensorflow.python.client import device_lib
print device_lib.list_local_devices()
conf_file = '/'.join(str.split(lsdc.__file__, '/')[:-3]) +\
'/tensorflow_data/autoenc/' + FLAGS.hyper + '/conf.py'
if not os.path.exists(conf_file):
sys.exit("Experiment configuration not found")
hyperparams = imp.load_source('hyperparams', conf_file)
conf = hyperparams.configuration
if FLAGS.visualize:
print 'creating visualizations ...'
conf = adapt_params_visualize(conf, FLAGS.visualize)
print '-------------------------------------------------------------------'
print 'verify current settings!! '
for key in conf.keys():
print key, ': ', conf[key]
print '-------------------------------------------------------------------'
if FLAGS.visualize:
visualize(conf, refeed_img= True)
# visualize(conf, refeed_img= False)
return
print 'Constructing models and inputs.'
with tf.variable_scope('model') as training_scope:
images, actions, states = build_tfrecord_input(conf, training=True)
images_val, actions_val, states_val = build_tfrecord_input(conf, training=False)
train_cond = tf.placeholder(tf.int32, shape=[], name="train_cond")
image, actions, states = tf.cond(train_cond > 0, # if 1 use trainigbatch else validation batch
lambda: [images, actions, states],
lambda: [images_val, actions_val, states_val])
model = Model(conf, image, actions, states)
print 'Constructing saver.'
# Make saver.
saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.VARIABLES), max_to_keep=0)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
# Make training session.
sess = tf.InteractiveSession(config= tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.train.SummaryWriter(
conf['output_dir'], graph=sess.graph, flush_secs=10)
tf.train.start_queue_runners(sess)
sess.run(tf.initialize_all_variables())
itr_0 =0
if conf['pretrained_model']: # is the order of initialize_all_variables() and restore() important?!?
saver.restore(sess, conf['pretrained_model'])
# resume training at iteration step of the loaded model:
import re
itr_0 = re.match('.*?([0-9]+)$', conf['pretrained_model']).group(1)
itr_0 = int(itr_0)
print 'resuming training at iteration: ', itr_0
tf.logging.info('iteration number, cost')
starttime = datetime.now()
t_iter = []
# Run training.
for itr in range(itr_0, conf['num_iterations'], 1):
t_startiter = datetime.now()
# Generate new batch of data_files.
feed_dict = {model.prefix: 'train',
model.iter_num: np.float32(itr),
model.lr: conf['learning_rate'],
train_cond: 1}
cost, _, summary_str = sess.run([model.loss, model.train_op, model.summ_op],
feed_dict)
# Print info: iteration #, cost.
if (itr) % 10 ==0:
tf.logging.info(str(itr) + ' ' + str(cost))
if (itr) % VAL_INTERVAL == 2:
# Run through validation set.
feed_dict = {model.lr: 0.0,
model.prefix: 'val',
model.iter_num: np.float32(itr),
train_cond: 0}
_, val_summary_str = sess.run([model.train_op, model.summ_op],
feed_dict)
summary_writer.add_summary(val_summary_str, itr)
if (itr) % SAVE_INTERVAL == 2:
tf.logging.info('Saving model to' + conf['output_dir'])
saver.save(sess, conf['output_dir'] + '/model' + str(itr))
t_iter.append((datetime.now() - t_startiter).seconds * 1e6 + (datetime.now() - t_startiter).microseconds )
if itr % 100 == 1:
hours = (datetime.now() -starttime).seconds/3600
tf.logging.info('running for {0}d, {1}h, {2}min'.format(
(datetime.now() - starttime).days,
hours,
(datetime.now() - starttime).seconds/60 - hours*60))
avg_t_iter = np.sum(np.asarray(t_iter))/len(t_iter)
tf.logging.info('time per iteration: {0}'.format(avg_t_iter/1e6))
tf.logging.info('expected for complete training: {0}h '.format(avg_t_iter /1e6/3600 * conf['num_iterations']))
if (itr) % SUMMARY_INTERVAL:
summary_writer.add_summary(summary_str, itr)
tf.logging.info('Saving model.')
saver.save(sess, conf['output_dir'] + '/model')
tf.logging.info('Training complete')
tf.logging.flush()
def main(unused_argv, conf_script= None):
if FLAGS.device != None:
os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS.device)
else:
os.environ["CUDA_VISIBLE_DEVICES"] = str(0)
print 'using CUDA_VISIBLE_DEVICES=', FLAGS.device
from tensorflow.python.client import device_lib
print device_lib.list_local_devices()
if conf_script == None: conf_file = FLAGS.hyper
else: conf_file = conf_script
if not os.path.exists(FLAGS.hyper):
sys.exit("Experiment configuration not found")
hyperparams = imp.load_source('hyperparams', conf_file)
conf = hyperparams.configuration
if FLAGS.visualize:
print 'creating visualizations ...'
conf = adapt_params_visualize(conf, FLAGS.visualize)
print '-------------------------------------------------------------------'
print 'verify current settings!! '
for key in conf.keys():
print key, ': ', conf[key]
print '-------------------------------------------------------------------'
print 'Constructing models and inputs...'
with tf.variable_scope('train_model', reuse=None) as training_scope:
model = Model(conf)
with tf.variable_scope('fwd_model', reuse=None):
fwd_conf = copy.deepcopy(conf)
fwd_conf['batch_size'] = conf['num_smp']
fwd_model = Model(fwd_conf, reuse_scope= training_scope)
train_images, train_actions, train_states = build_tfrecord_input(conf, training=True)
val_images, val_actions, val_states = build_tfrecord_input(conf, training=False)
print 'Constructing saver.'
# Make saver.
saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.VARIABLES), max_to_keep=0)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
# Make training session.
sess = tf.InteractiveSession(config= tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.train.SummaryWriter(
conf['output_dir'], graph=sess.graph, flush_secs=10)
tf.train.start_queue_runners(sess)
sess.run(tf.initialize_all_variables())
if conf['visualize']:
saver.restore(sess, conf['visualize'])
itr = 0
## visualize the videos with worst score !!!
videos, states, actions, bestnoise, worstnoise = run_foward_passes(conf, fwd_model,
val_images,
val_states,
val_actions, sess, itr)
feed_dict = {model.images: videos,
model.states: states,
model.actions: actions,
model.noise: bestnoise,
model.prefix: 'visual',
model.iter_num: 0,
model.lr: 0,
}
gen_images, mask_list = sess.run([model.gen_images, model.gen_masks], feed_dict)
file_path = conf['output_dir']
cPickle.dump(gen_images, open(file_path + '/gen_image_seq.pkl','wb'))
cPickle.dump(videos, open(file_path + '/ground_truth.pkl', 'wb'))
cPickle.dump(mask_list, open(file_path + '/mask_list.pkl', 'wb'))
print 'written files to:' + file_path
trajectories = video_prediction.utils_vpred.create_gif.comp_video(conf['output_dir'], conf, suffix='_best')
video_prediction.utils_vpred.create_gif.comp_masks(conf['output_dir'], conf, trajectories, suffix='_best')
### visualizing videos with highest cost noise:
feed_dict = {model.images: videos,
model.states: states,
model.actions: actions,
model.noise: worstnoise,
model.prefix: 'visual',
model.iter_num: 0,
model.lr: 0,
}
gen_images, mask_list = sess.run([model.gen_images, model.gen_masks], feed_dict)
file_path = conf['output_dir']
cPickle.dump(gen_images, open(file_path + '/gen_image_seq.pkl', 'wb'))
cPickle.dump(videos, open(file_path + '/ground_truth.pkl', 'wb'))
cPickle.dump(mask_list, open(file_path + '/mask_list.pkl', 'wb'))
print 'written files to:' + file_path
trajectories = video_prediction.utils_vpred.create_gif.comp_video(conf['output_dir'], conf, suffix='_worst')
video_prediction.utils_vpred.create_gif.comp_masks(conf['output_dir'], conf, trajectories, suffix='_worst')
return
itr_0 =0
if conf['pretrained_model']: # is the order of initialize_all_variables() and restore() important?!?
saver.restore(sess, conf['pretrained_model'])
# resume training at iteration step of the loaded model:
import re
itr_0 = re.match('.*?([0-9]+)$', conf['pretrained_model']).group(1)
itr_0 = int(itr_0)
print 'resuming training at iteration: ', itr_0
tf.logging.info('iteration number, cost')
starttime = datetime.now()
t_iter = []
# Run training.
for itr in range(itr_0, conf['num_iterations'], 1):
t_startiter = datetime.now()
# Generate new batch of data_files.
videos, states, actions, bestnoise, worstnoise = run_foward_passes(conf, fwd_model,
train_images,
train_states,
train_actions, sess, itr)
feed_dict = {model.images: videos,
model.states: states,
model.actions: actions,
model.noise: bestnoise,
model.prefix: 'train',
model.iter_num: np.float32(itr),
model.lr: conf['learning_rate'],
}
cost, _, summary_str = sess.run([model.loss, model.train_op, model.summ_op],
feed_dict)
# Print info: iteration #, cost.
if (itr) % 10 ==0:
tf.logging.info(str(itr) + ' ' + str(cost))
if (itr) % VAL_INTERVAL == 2:
videos, states, actions, bestnoise, worstnoise = run_foward_passes(conf, fwd_model,
val_images,
val_states,
val_actions, sess, itr)
feed_dict = {model.images: videos,
model.states: states,
model.actions: actions,
model.noise: bestnoise,
model.prefix: 'val',
model.iter_num: np.float32(itr),
model.lr: 0.0,
}
_, val_summary_str = sess.run([model.train_op, model.summ_op],
feed_dict)
summary_writer.add_summary(val_summary_str, itr)
if (itr) % SAVE_INTERVAL == 2:
tf.logging.info('Saving model to' + conf['output_dir'])
saver.save(sess, conf['output_dir'] + '/model' + str(itr))
t_iter.append((datetime.now() - t_startiter).seconds * 1e6 + (datetime.now() - t_startiter).microseconds )
if itr % 100 == 1:
hours = (datetime.now() -starttime).seconds/3600
tf.logging.info('running for {0}d, {1}h, {2}min'.format(
(datetime.now() - starttime).days,
hours,
(datetime.now() - starttime).seconds/60 - hours*60))
avg_t_iter = np.sum(np.asarray(t_iter))/len(t_iter)
tf.logging.info('time per iteration: {0}'.format(avg_t_iter/1e6))
tf.logging.info('expected for complete training: {0}h '.format(avg_t_iter /1e6/3600 * conf['num_iterations']))
if (itr) % SUMMARY_INTERVAL:
summary_writer.add_summary(summary_str, itr)
tf.logging.info('Saving model.')
saver.save(sess, conf['output_dir'] + '/model')
tf.logging.info('Training complete')
tf.logging.flush()
def main(unused_argv):
os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS.device)
print 'using CUDA_VISIBLE_DEVICES=', FLAGS.device
from tensorflow.python.client import device_lib
print device_lib.list_local_devices()
conf_file = FLAGS.hyper
if not os.path.exists(conf_file):
sys.exit("Experiment configuration not found")
hyperparams = imp.load_source('hyperparams', conf_file)
conf = hyperparams.configuration
conf['event_log_dir'] = conf['output_dir']
if FLAGS.visualize:
print 'creating visualizations ...'
conf['data_dir'] = '/'.join(
str.split(conf['data_dir'], '/')[:-1] + ['test'])
conf['visualize'] = conf['output_dir'] + '/' + FLAGS.visualize
conf['event_log_dir'] = '/tmp'
filenames = gfile.Glob(os.path.join(conf['data_dir'], '*'))
conf['visual_file'] = filenames
conf['batch_size'] = 18
print 'Constructing models and inputs.'
with tf.variable_scope('trainmodel') as training_scope:
images, actions, states, poses = build_tfrecord_input(conf,
training=True)
model = Model(conf, images, poses)
with tf.variable_scope('val_model', reuse=None):
images_val, actions_val, states_val, poses_val = build_tfrecord_input(
conf, training=False)
val_model = Model(conf,
images_val,
poses_val,
reuse_scope=training_scope)
print 'Constructing saver.'
# Make saver.
saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.VARIABLES),
max_to_keep=0)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
# Make training session.
sess = tf.InteractiveSession(config=tf.ConfigProto(
gpu_options=gpu_options))
summary_writer = tf.train.SummaryWriter(conf['event_log_dir'],
graph=sess.graph,
flush_secs=10)
tf.train.start_queue_runners(sess)
sess.run(tf.initialize_all_variables())
print '-------------------------------------------------------------------'
print 'verify current settings!! '
for key in conf.keys():
print key, ': ', conf[key]
print '-------------------------------------------------------------------'
if FLAGS.visualize:
visualize(conf, sess, saver, val_model)
return
itr_0 = 0
if FLAGS.pretrained != None:
conf['pretrained_model'] = FLAGS.pretrained
saver.restore(sess, conf['pretrained_model'])
# resume training at iteration step of the loaded model:
import re
itr_0 = re.match('.*?([0-9]+)$', conf['pretrained_model']).group(1)
itr_0 = int(itr_0)
print 'resuming training at iteration: ', itr_0
tf.logging.info('iteration number, cost')
starttime = datetime.now()
t_iter = []
# Run training.
for itr in range(itr_0, conf['num_iterations'], 1):
t_startiter = datetime.now()
# Generate new batch of data_files.
feed_dict = {
model.prefix: 'train',
model.iter_num: np.float32(itr),
model.lr: conf['learning_rate'],
}
cost, _, summary_str = sess.run(
[model.loss, model.train_op, model.summ_op], feed_dict)
# Print info: iteration #, cost.
if (itr) % 10 == 0:
tf.logging.info(str(itr) + ' ' + str(cost))
if (itr) % VAL_INTERVAL == 2:
# Run through validation set.
feed_dict = {
val_model.lr: 0.0,
val_model.prefix: 'val',
val_model.iter_num: np.float32(itr),
}
_, val_summary_str = sess.run(
[val_model.train_op, val_model.summ_op], feed_dict)
summary_writer.add_summary(val_summary_str, itr)
if (itr) % SAVE_INTERVAL == 2:
tf.logging.info('Saving model to' + conf['output_dir'])
oldfile = conf['output_dir'] + '/model' + str(itr - SAVE_INTERVAL)
if os.path.isfile(oldfile):
os.system("rm {}".format(oldfile))
os.system("rm {}".format(oldfile + '.meta'))
saver.save(sess, conf['output_dir'] + '/model' + str(itr))
t_iter.append((datetime.now() - t_startiter).seconds * 1e6 +
(datetime.now() - t_startiter).microseconds)
if itr % 100 == 1:
hours = (datetime.now() - starttime).seconds / 3600
tf.logging.info('running for {0}d, {1}h, {2}min'.format(
(datetime.now() - starttime).days, hours,
(datetime.now() - starttime).seconds / 60 - hours * 60))
avg_t_iter = np.sum(np.asarray(t_iter)) / len(t_iter)
tf.logging.info('time per iteration: {0}'.format(avg_t_iter / 1e6))
tf.logging.info('expected for complete training: {0}h '.format(
avg_t_iter / 1e6 / 3600 * conf['num_iterations']))
if (itr) % SUMMARY_INTERVAL:
summary_writer.add_summary(summary_str, itr)
tf.logging.info('Saving model.')
saver.save(sess, conf['output_dir'] + '/model')
tf.logging.info('Training complete')
tf.logging.flush()
def visualize(conf):
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
# Make training session.
sess = tf.InteractiveSession(config=tf.ConfigProto(
gpu_options=gpu_options))
tf.train.start_queue_runners(sess)
input_distrib = tf.placeholder(tf.float32,
shape=(conf['batch_size'],
conf['num_objects'], 64, 64))
images = tf.placeholder(tf.float32,
name='images',
shape=(conf['batch_size'], conf['sequence_length'],
64, 64, 3))
with tf.variable_scope('model', reuse=None):
val_images, _, _, object_pos = build_tfrecord_input(conf,
training=False)
model = CorrectorModel(conf, images, pix_distrib=input_distrib)
sess.run(tf.initialize_all_variables())
saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.VARIABLES),
max_to_keep=0)
saver.restore(sess, conf['output_dir'] + '/' + FLAGS.visualize)
ground_truth, object_pos_npy = sess.run([val_images, object_pos])
object_pos_npy = object_pos_npy[:, 0] #take first time step only
one_hot_img = np.zeros([conf['batch_size'], conf['num_objects'], 64, 64])
for ob in range(4):
for b in range(conf['batch_size']):
coords = mujoco_to_imagespace(object_pos_npy[b, ob])
one_hot_img[b, ob, coords[0], coords[1]] = 1
output_distrib_list, gen_masks_list, gen_image_list = [], [], []
for t in range(conf['sequence_length'] - 1):
if t == 0:
_one_hot_images = one_hot_img
else:
_one_hot_images = output_distrib
feed_dict = {
model.prefix: 'ctrl',
model.lr: 0,
images:
ground_truth[:, t:t + 2], #could alternatively feed in gen_image
input_distrib: _one_hot_images
}
gen_image, gen_masks, output_distrib = sess.run(
[model.gen_images, model.gen_masks, model.gen_distrib], feed_dict)
combine_obj = np.zeros([conf['batch_size'], 64, 64])
for i in conf['num_objects']:
combine_obj += output_distrib[:, i]
output_distrib_list.append(combine_obj)
gen_image_list.append(gen_image)
gen_masks_list.append(gen_masks)
file_path = conf['output_dir']
cPickle.dump(gen_image_list, open(file_path + '/gen_image_seq.pkl', 'wb'))
cPickle.dump(ground_truth, open(file_path + '/ground_truth.pkl', 'wb'))
cPickle.dump(gen_masks_list, open(file_path + '/mask_list.pkl', 'wb'))
cPickle.dump(output_distrib_list,
open(file_path + '/output_distrib_list.pkl', 'wb'))
print 'written files to:' + file_path
trajectories = make_video(conf['output_dir'], conf)
video_prediction.utils_vpred.create_gif.comp_masks(conf['output_dir'],
conf, trajectories)
return