def main(): # capture the config path from the run arguments # then process the json configuration file try: args = get_args() config = process_config(args.config) config.old_tfrecords = args.old_tfrecords config.normalize_data = False except Exception as e: print("An error occurred during processing the configuration file") print(e) exit(0) # create the experiments dirs create_dirs( [config.summary_dir, config.checkpoint_dir, config.config_file_dir]) # create tensorflow session sess = tf.Session() # create your data generator #train_data = DataGenerator(config, sess, train=True) test_data = DataGenerator(config, sess, train=False) next_element = test_data.get_next_batch() dir_name = "test" while True: try: features = sess.run(next_element) features = convert_dict_to_list_subdicts(features, config.test_batch_size) features = features[0] summaries = create_target_summary_dicts(features) dir_path = check_exp_folder_exists_and_create(features, dir_name) if dir_path is not None: export_summary_images(config=config, summaries_dict_images=summaries, dir_path=dir_path) except tf.errors.OutOfRangeError: print("done exporting") break
def main(): # create tensorflow session sess = tf.Session() try: args = get_args() config = process_config(args.config) config.old_tfrecords = args.old_tfrecords config.normalize_data = args.normalize_data except Exception as e: print("An error occurred during processing the configuration file") print(e) exit(0) # create the experiments dirs create_dirs( [config.summary_dir, config.checkpoint_dir, config.config_file_dir]) # create your data generator train_data = DataGenerator(config, sess, train=True) test_data = DataGenerator(config, sess, train=False) logger = Logger(sess, config) print("using {} rollout steps".format(config.n_rollouts)) inp_rgb = tf.placeholder("float", [None, 120, 160, 7]) control = tf.placeholder("float", [None, 6]) gt_seg = tf.placeholder("float", [None, 120, 160]) pred = cnnmodel(inp_rgb, control) predictions = tf.reshape( pred, [-1, pred.get_shape()[1] * pred.get_shape()[2]]) labels = tf.reshape(gt_seg, [-1, gt_seg.get_shape()[1] * gt_seg.get_shape()[2]]) global_step_tensor = tf.Variable(0, trainable=False, name='global_step') loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=predictions)) optimizer = tf.train.AdamOptimizer( learning_rate=config.learning_rate).minimize( loss, global_step=global_step_tensor) with tf.variable_scope('cur_epoch'): cur_epoch_tensor = tf.Variable(0, trainable=False, name='cur_epoch') increment_cur_epoch_tensor = tf.assign(cur_epoch_tensor, cur_epoch_tensor + 1) with tf.variable_scope('global_step'): cur_batch_tensor = tf.Variable(0, trainable=False, name='cur_batch') increment_cur_batch_tensor = tf.assign(cur_batch_tensor, cur_batch_tensor + 1) next_element_train = train_data.get_next_batch() next_element_test = test_data.get_next_batch() init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init) saver = tf.train.Saver(max_to_keep=config.max_checkpoints_to_keep) latest_checkpoint = tf.train.latest_checkpoint(config.checkpoint_dir) if latest_checkpoint: print("Loading model checkpoint {} ...\n".format(latest_checkpoint)) saver.restore(sess, latest_checkpoint) print("Model loaded") def _process_rollouts(feature, train=True): gt_merged_seg_rollout_batch = [] input_merged_images_rollout_batch = [] gripper_pos_vel_rollout_batch = [] for step in range(config.n_rollouts - 1): if step < feature["unpadded_experiment_length"]: obj_segments = feature["object_segments"][step] """ transform (3,120,160,7) into (1,120,160,7) by merging the rgb,depth and seg masks """ input_merged_images = create_full_images_of_object_masks( obj_segments) obj_segments_gt = feature["object_segments"][step + 1] gt_merged_seg = create_full_images_of_object_masks( obj_segments_gt)[:, :, 3] gripper_pos = feature["gripperpos"][step + 1] gripper_vel = feature["grippervel"][step + 1] gripper_pos_vel = np.concatenate([gripper_pos, gripper_vel]) gt_merged_seg_rollout_batch.append(gt_merged_seg) input_merged_images_rollout_batch.append(input_merged_images) gripper_pos_vel_rollout_batch.append(gripper_pos_vel) if train: retrn = sess.run( [optimizer, loss, pred], feed_dict={ inp_rgb: input_merged_images_rollout_batch, control: gripper_pos_vel_rollout_batch, gt_seg: gt_merged_seg_rollout_batch }) return retrn[1], retrn[2] else: retrn = sess.run( [loss, pred], feed_dict={ inp_rgb: input_merged_images_rollout_batch, control: gripper_pos_vel_rollout_batch, gt_seg: gt_merged_seg_rollout_batch }) """ sigmoid cross entropy runs logits through sigmoid but only during train time """ seg_data = sigmoid(retrn[1]) seg_data[seg_data >= 0.5] = 1.0 seg_data[seg_data < 0.5] = 0.0 return retrn[0], seg_data, gt_merged_seg_rollout_batch for cur_epoch in range(cur_epoch_tensor.eval(sess), config.n_epochs + 1, 1): while True: try: features = sess.run(next_element_train) features = convert_dict_to_list_subdicts( features, config.train_batch_size) loss_batch = [] sess.run(increment_cur_batch_tensor) for _ in range(config.train_batch_size): for feature in features: loss_train, _ = _process_rollouts(feature) loss_batch.append([loss_train]) cur_batch_it = cur_batch_tensor.eval(sess) loss_mean_batch = np.mean(loss_batch) print('train loss batch {0:} is: {1:.4f}'.format( cur_batch_it, loss_mean_batch)) summaries_dict = {config.exp_name + '_loss': loss_mean_batch} logger.summarize(cur_batch_it, summaries_dict=summaries_dict, summarizer="train") if cur_batch_it % config.test_interval == 1: print("Executing test batch") features_idx = 0 # always take first element for testing features = sess.run(next_element_test) features = convert_dict_to_list_subdicts( features, config.test_batch_size) loss_test_batch = [] for i in range(config.test_batch_size): loss_test, seg_data, gt_seg_data = _process_rollouts( features[features_idx], train=False) loss_test_batch.append(loss_test) loss_test_mean_batch = np.mean(loss_test_batch) summaries_dict = { config.exp_name + '_test_loss': loss_test_mean_batch } logger.summarize(cur_batch_it, summaries_dict=summaries_dict, summarizer="test") print('test loss is: {0:.4f}'.format(loss_test_mean_batch)) if seg_data is not None and gt_seg_data is not None: """ create gif here """ create_seg_gif(features, features_idx, config, seg_data, gt_seg_data, dir_name="tests_during_training", cur_batch_it=cur_batch_it) if cur_batch_it % config.model_save_step_interval == 1: print("Saving model...") saver.save(sess, config.checkpoint_dir, global_step_tensor) print("Model saved") except tf.errors.OutOfRangeError: break sess.run(increment_cur_epoch_tensor) return None