def read_data_from_file(self, filename):
field_dict = {self._input_field_name: False, self._target_field_name: False}
data = hdf5_utils.read_hdf5_file_to_numpy_dict(filename, field_dict)
if self._input_field_name is not None:
assert(field_dict[self._input_field_name])
if self._target_field_name is not None:
assert(field_dict[self._target_field_name])
return data
def read_samples_from_file(filename):
stacked_samples, attr_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(filename, read_attributes=True)
samples = []
for key in stacked_samples:
for i in range(len(stacked_samples[key])):
if len(samples) <= i:
samples.append({})
samples[i][key] = stacked_samples[key][i, ...]
return samples, attr_dict
def run(args):
if args.input_path is not None:
input_path = args.input_path
input_files = list(
file_helpers.input_filename_generator_hdf5(
input_path, file_helpers.DEFAULT_HDF5_PATTERN))
else:
input_list_file = args.input_list_file
with file(input_list_file, "r") as fin:
input_files = [l.strip() for l in fin.readlines()]
dataset_kwargs = {}
if args.compression:
dataset_kwargs.update({"compression": args.compression})
if args.compression_level >= 0:
dataset_kwargs.update({"compression_opts": args.compression_level})
print("Counting {} input files".format(len(input_files)))
for i, input_file in enumerate(input_files):
print("Reading input file #{} out of {}".format(i, len(input_files)))
field_dict = None
data, attr_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(
input_file, field_dict, read_attributes=True)
print("Writing {} samples with new compression settings".format(
data[list(data.keys())[0]].shape[0]))
hdf5_utils.write_numpy_dict_to_hdf5_file(input_file + "_recompressed",
data, attr_dict,
**dataset_kwargs)
if args.check_written_samples:
print("Reading samples from file {}".format(input_file +
"_recompressed"))
written_data, written_attr_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(
input_file + "_recompressed", field_dict, read_attributes=True)
for key in data:
assert (np.all(data[key] == written_data[key]))
for key in attr_dict:
assert (np.all(attr_dict[key] == written_attr_dict[key]))
os.remove(input_file)
os.rename(input_file + "_recompressed", input_file)
def run(args):
if args.input_path is not None:
input_path = args.input_path
input_files = list(
file_helpers.input_filename_generator_hdf5(
input_path, file_helpers.DEFAULT_HDF5_PATTERN))
else:
input_list_file = args.input_list_file
with open(input_list_file, "r") as fin:
input_files = [l.strip() for l in fin.readlines()]
print("Counting {} input files".format(len(input_files)))
# Count total number of samples
total_num_samples = 0
for i, input_file in enumerate(input_files):
print("Reading input file #{} out of {} ({})".format(
i, len(input_files), input_file))
field_dict = {"scores": False}
data, attr = hdf5_utils.read_hdf5_file_to_numpy_dict(
input_file, field_dict, read_attributes=True)
total_num_samples += data["scores"].shape[0]
print("Total number of samples: {}".format(total_num_samples))
def run(args):
# Create environment
client_id = args.client_id
with open(args.environment_config, "r") as fin:
environment_config = yaml.load(fin)
environment = env_factory.create_environment_from_config(
environment_config, client_id, use_openai_wrapper=True)
result = environment.base.get_mapper().perform_info()
map_resolution = result.resolution
axis_mode = environment_config["collect_data"]["axis_mode"]
forward_factor = float(
environment_config["collect_data"]["forward_factor"])
downsample_to_grid = environment_config["collect_data"][
"downsample_to_grid"]
raycast_max_range = float(environment_config["octomap"]["max_range"])
logger.info("map_resolution={}".format(map_resolution))
logger.info("axis_mode={}".format(axis_mode))
logger.info("forward_factor={}".format(forward_factor))
logger.info("downsample_to_grid={}".format(downsample_to_grid))
logger.info("raycast_max_range={}".format(raycast_max_range))
environment.base.get_engine().disable_input()
pose_list = []
def before_reset_hook(env):
pose = pose_list[0]
print("Resetting episode with pose {}".format(pose))
return pose
def after_reset_hook(env):
logger.info("Env reset in pose {}".format(env.base.get_pose()))
environment.base.before_reset_hooks.register(before_reset_hook,
environment)
environment.base.after_reset_hooks.register(after_reset_hook, environment)
input_path = os.path.dirname(args.input_filename_prefix)
print("Input path: {}".format(input_path))
input_filename_pattern = "{:s}_(\d+)\.hdf5".format(
os.path.basename(args.input_filename_prefix))
input_filenames_and_matches = file_utils.get_matching_filenames(
input_filename_pattern, path=input_path, return_match_objects=True)
print("Number of input files: {}".format(len(input_filenames_and_matches)))
only_episode = args.episode
max_steps = args.max_steps
for i, (input_filename,
filename_match) in enumerate(input_filenames_and_matches):
input_episode_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(
os.path.join(input_path, input_filename))
episode_poses = [environment.base.Pose(location, orientation_rpy) for location, orientation_rpy \
in zip(input_episode_dict["location"], input_episode_dict["orientation_rpy"])]
if max_steps is not None:
episode_poses = episode_poses[:max_steps]
del pose_list[:]
pose_list.extend(episode_poses)
episode = int(filename_match.group(1))
if only_episode is not None and episode != only_episode:
continue
logger.info("Running episode #{} from input file {}".format(
episode, input_filename))
run_episode(environment,
pose_list,
downsample_to_grid=downsample_to_grid,
measure_timing=args.measure_timing)
episode += 1
def run(args):
# Read config file
topic_cmdline_mappings = {"tensorflow": "tf"}
topics = ["tensorflow", "io", "training", "data"]
cfg = configuration.get_config_from_cmdline(args, topics,
topic_cmdline_mappings)
if args.config is not None:
with open(args.config, "r") as config_file:
tmp_cfg = yaml.load(config_file)
configuration.update_config_from_other(cfg, tmp_cfg)
# Read model config
if "model" in cfg:
model_config = cfg["model"]
elif args.model_config is not None:
model_config = {}
else:
logger.fatal(
"ERROR: Model configuration must be in general config file or provided in extra config file."
)
import sys
sys.exit(1)
if args.model_config is not None:
with open(args.model_config, "r") as config_file:
tmp_model_config = yaml.load(config_file)
configuration.update_config_from_other(model_config,
tmp_model_config)
cfg = AttributeDict.convert_deep(cfg)
model_config = AttributeDict.convert_deep(model_config)
if args.hdf5_data_stats_path is not None:
logger.info("Loading data stats from HDF5 file")
data_stats_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(
args.hdf5_data_stats_path)
else:
data_stats_dict = None
if args.use_train_data or data_stats_dict is None:
logger.info("Creating train dataflow")
train_dataflow = input_pipeline.InputAndTargetDataFlow(
cfg.data.train_path,
cfg.data,
shuffle_lmdb=args.shuffle,
override_data_stats=data_stats_dict,
verbose=True)
data_stats_dict = train_dataflow.get_data_stats()
if args.use_train_data:
dataflow = train_dataflow
if not args.use_train_data:
assert cfg.data.test_path is not None, "Test data path has to be specified if not using train data"
logger.info("Creating test dataflow")
dataflow = input_pipeline.InputAndTargetDataFlow(
cfg.data.test_path,
cfg.data,
shuffle_lmdb=args.shuffle,
override_data_stats=data_stats_dict,
verbose=True)
logger.info("# samples in dataset: {}".format(dataflow.size()))
logger.info("Input and target shapes:")
dataflow.reset_state()
first_sample = next(dataflow.get_data())
tensor_shapes = [tensor.shape for tensor in first_sample]
tensor_dtypes = [tensor.dtype for tensor in first_sample]
logger.info(" Shape of input: {}".format(first_sample[0].shape))
logger.info(" Type of input: {}".format(first_sample[0].dtype))
logger.info(" Shape of target: {}".format(first_sample[1].shape))
logger.info(" Type of target: {}".format(first_sample[1].dtype))
# Create tensorflow session
logger.info("Creating tensorflow session")
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.tensorflow.gpu_memory_fraction)
tf_config = tf.ConfigProto(gpu_options=gpu_options)
tf_config.intra_op_parallelism_threads = cfg.tensorflow.intra_op_parallelism
tf_config.inter_op_parallelism_threads = cfg.tensorflow.inter_op_parallelism
tf_config.log_device_placement = cfg.tensorflow.log_device_placement
with tf.Session(config=tf_config) as sess:
coord = tf.train.Coordinator()
# def signal_handler(signal, frame):
# sess.close()
#
# batch_size = 1
#
# sample_stats = dataflow.get_sample_stats()
# pipeline = input_pipeline.TFDataFlowPipeline(
# dataflow.get_batch_dataflow(), tensor_shapes, tensor_dtypes, sess, coord, batch_size,
# cfg.tensorflow, is_training=False, sample_stats=sample_stats, is_batch_dataflow=True)
#
# Create model
#
# with tf.device("/gpu:0"):
# with tf.variable_scope("model"):
# model = models.Model(model_config,
# pipeline.tensors_batch[0],
# pipeline.tensors[1],
# is_training=False,
# verbose=args.verbose)
input_placeholder = tf.placeholder(dtype=tensor_dtypes[0],
shape=(1, ) + tensor_shapes[0],
name="Input")
target_placeholder = tf.placeholder(dtype=tensor_dtypes[1],
shape=(1, ) + tensor_shapes[1],
name="Target")
gpu_device_name = tf_utils.gpu_device_name()
print(tf_utils.get_available_cpu_ids(),
tf_utils.get_available_cpu_names())
print(tf_utils.get_available_gpu_ids(),
tf_utils.get_available_gpu_names())
print(gpu_device_name)
with tf.device(gpu_device_name):
with tf.variable_scope("model"):
model = models.Model(model_config,
input_placeholder,
target_placeholder,
is_training=False,
verbose=args.verbose)
try:
saver = tf.train.Saver(model.global_variables)
if args.check_numerics:
# Run numeric checks on all model checkpoints
if args.checkpoint is None:
ckpt = tf.train.get_checkpoint_state(args.model_dir)
checkpoint_paths = ckpt.all_model_checkpoint_paths
else:
checkpoint_path = os.path.join(args.model_dir, args.checkpoint)
checkpoint_paths = [checkpoint_path]
for checkpoint_path in checkpoint_paths:
if args.verbose:
logger.info(
"Checking numerics on model checkpoint {}".format(
checkpoint_path))
saver.restore(sess, checkpoint_path)
for var in model.variables:
if args.verbose:
logger.info(" Checking tensor {}".format(var.name))
sess.run(
tf.check_numerics(
var, "Numeric check for tensor {} failed".format(
var.name)))
return
# Restore model
if args.checkpoint is None:
logger.info("Reading latest checkpoint from {}".format(
args.model_dir))
ckpt = tf.train.get_checkpoint_state(args.model_dir)
if ckpt is None:
raise IOError("No previous checkpoint found at {}".format(
args.model_dir))
else:
logger.info('Found previous checkpoint... restoring')
checkpoint_path = ckpt.model_checkpoint_path
saver.restore(sess, checkpoint_path)
else:
checkpoint_path = os.path.join(args.model_dir, args.checkpoint)
if checkpoint_path is not None:
logger.info("Trying to restore model from checkpoint {}".format(
checkpoint_path))
saver.restore(sess, checkpoint_path)
# custom_threads = []
# pipeline.start()
# custom_threads.extend(pipeline.threads)
# # Start data provider threads
# custom_threads.extend(tf.train.start_queue_runners(sess=sess))
sess.graph.finalize()
# Running statistics
stats = None
denorm_target_list = []
denorm_output_list = []
logger.info("Starting evaluating")
for i, (input, target) in enumerate(dataflow.get_data()):
if args.verbose:
logger.info(" sample # {}".format(i))
if stats is None:
stats = AttributeDict()
stats.output = math_utils.SinglePassStatistics(target.shape)
stats.target = math_utils.SinglePassStatistics(target.shape)
stats.diff = math_utils.SinglePassStatistics(target.shape)
stats.squared_diff = math_utils.SinglePassStatistics(
target.shape)
stats.loss = math_utils.SinglePassStatistics(target.shape)
denorm_target = dataflow.input_and_target_retriever.denormalize_target(
target)
input_batch = input[np.newaxis, ...]
target_batch = target[np.newaxis, ...]
loss_v, loss_min_v, loss_max_v, output_batch = sess.run(
[model.loss, model.loss_min, model.loss_max, model.output],
feed_dict={
input_placeholder: input_batch,
target_placeholder: target_batch
})
output = output_batch[0, ...]
denorm_output = dataflow.input_and_target_retriever.denormalize_target(
output)
diff = denorm_output - denorm_target
squared_diff = np.square(diff)
if args.verbose:
logger.info("Output={}, Target={}, Diff={}, Diff^2={}".format(
denorm_output, denorm_target, diff, squared_diff))
logger.info(" loss: {}, min loss: {}, max loss: {}".format(
loss_v, loss_min_v, loss_max_v))
if diff > 80:
import time
time.sleep(5)
# Update stats
stats.output.add_value(denorm_output)
stats.target.add_value(denorm_target)
stats.diff.add_value(diff)
stats.squared_diff.add_value(squared_diff)
stats.loss.add_value(loss_v)
denorm_output_list.append(denorm_output)
denorm_target_list.append(denorm_target)
if i % 100 == 0 and i > 0:
logger.info("-----------")
logger.info("Statistics after {} samples:".format(i + 1))
for key in stats:
logger.info(
" {:s}: mean={:.4f}, stddev={:.4f}, min={:.4f}, max={:.4f}"
.format(key, stats[key].mean[0], stats[key].stddev[0],
float(stats[key].min), float(stats[key].max)))
logger.info("-----------")
import scipy.stats
correlation, pvalue = scipy.stats.pearsonr(
np.array(denorm_target_list), np.array(denorm_output_list))
logger.info("Pearson correlation: {} [p={}]".format(
correlation, pvalue))
correlation, pvalue = scipy.stats.spearmanr(
np.array(denorm_target_list), np.array(denorm_output_list))
logger.info("Spearman correlation: {} [p={}]".format(
correlation, pvalue))
obj = {
"a": np.array(denorm_target_list),
"b": np.array(denorm_output_list)
}
np.savez("spearman.npz", **obj)
hdf5_utils.write_numpy_dict_to_hdf5_file("spearman.hdf5", obj)
if args.visualize:
import visualization
fig = 1
fig = visualization.plot_grid(input[..., 2],
input[..., 3],
title_prefix="input",
show=False,
fig_offset=fig)
# fig = visualization.plot_grid(record.in_grid_3d[..., 6], record.in_grid_3d[..., 7], title_prefix="in_grid_3d", show=False, fig_offset=fig)
visualization.show(stop=True)
except Exception as exc:
logger.info("Exception in evaluation oop: {}".format(exc))
traceback.print_exc()
coord.request_stop(exc)
raise exc
finally:
logger.info("Requesting stop")
coord.request_stop()
# pipeline.stop()
# coord.join(custom_threads, stop_grace_period_secs=(2 * cfg.io.timeout))
sess.close()
def run(args):
# Create environment
client_id = args.client_id
with open(args.environment_config, "r") as fin:
environment_config = yaml.load(fin)
environment = env_factory.create_environment_from_config(
environment_config, client_id, use_openai_wrapper=True)
result = environment.base.get_mapper().perform_info()
map_resolution = result.resolution
axis_mode = environment_config["collect_data"]["axis_mode"]
forward_factor = float(
environment_config["collect_data"]["forward_factor"])
downsample_to_grid = environment_config["collect_data"][
"downsample_to_grid"]
raycast_max_range = float(environment_config["octomap"]["max_range"])
logger.info("map_resolution={}".format(map_resolution))
logger.info("axis_mode={}".format(axis_mode))
logger.info("forward_factor={}".format(forward_factor))
logger.info("downsample_to_grid={}".format(downsample_to_grid))
logger.info("raycast_max_range={}".format(raycast_max_range))
environment.base.get_engine().disable_input()
pose_list = []
def before_reset_hook(env):
pose = pose_list[0]
print("Resetting episode with pose {}".format(pose))
return pose
def after_reset_hook(env):
logger.info("Env reset in pose {}".format(env.base.get_pose()))
environment.base.before_reset_hooks.register(before_reset_hook,
environment)
environment.base.after_reset_hooks.register(after_reset_hook, environment)
input_path = os.path.dirname(args.input_filename_prefix)
print("Input path: {}".format(input_path))
input_filename_pattern = "{:s}_(\d+)\.hdf5".format(
os.path.basename(args.input_filename_prefix))
input_filenames_and_matches = file_utils.get_matching_filenames(
input_filename_pattern, path=input_path, return_match_objects=True)
print("Number of input files: {}".format(len(input_filenames_and_matches)))
for i, (input_filename,
filename_match) in enumerate(input_filenames_and_matches):
input_episode_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(
os.path.join(input_path, input_filename))
del pose_list[:]
pose_list.extend([environment.base.Pose(location, orientation_rpy) for location, orientation_rpy \
in zip(input_episode_dict["location"], input_episode_dict["orientation_rpy"])])
episode = int(filename_match.group(1))
if args.output_filename_prefix:
hdf5_filename = "{:s}_{:d}.hdf5".format(
args.output_filename_prefix, episode)
logger.info("File name for episode {}: {}".format(
episode, hdf5_filename))
if os.path.isfile(hdf5_filename):
logger.info("File '{}' already exists. Skipping.".format(
hdf5_filename))
continue
logger.info("Running episode #{} from input file {}".format(
episode, input_filename))
output = run_episode(episode,
environment,
input_episode_dict,
args.reset_interval,
args.reset_score_threshold,
downsample_to_grid=downsample_to_grid,
measure_timing=args.measure_timing)
if args.output_filename_prefix:
locations = [pose.location() for pose in output["pose"]]
orientation_rpys = [
pose.orientation_rpy() for pose in output["pose"]
]
hdf5_dict = {
"score": np.asarray(output["score"]),
"computed_reward": np.asarray(output["computed_reward"]),
"true_reward": np.asarray(output["true_reward"]),
"location": np.asarray(locations),
"orientation_rpy": np.asarray(orientation_rpys),
}
if os.path.isfile(hdf5_filename):
raise RuntimeError(
"ERROR: Output file '{}' already exists".format(
hdf5_filename))
if not args.dry_run:
hdf5_utils.write_numpy_dict_to_hdf5_file(
hdf5_filename, hdf5_dict)
logger.info(
"Wrote output to HDF5 file: {}".format(hdf5_filename))
episode += 1
def read_data_statistics(stats_filename):
statistics_dict = hdf5_utils.read_hdf5_file_to_numpy_dict(stats_filename)
return statistics_dict
