def run(args): filename_generator = file_helpers.input_filename_generator_hdf5(args.hdf5_path) filenames = list(filename_generator) if not os.path.isdir(args.tfrecords_path): os.makedirs(args.tfrecords_path) for filename in filenames: hdf5_record_batch = data_record.read_hdf5_records_v2(filename) output_filename = os.path.join(args.tfrecords_path, os.path.basename(filename[:-len('.hdf5')]) + '.tfrecords') print(output_filename) tfrecords_writer = tf.python_io.TFRecordWriter(output_filename) for example in data_record.generate_tfrecords_from_batch(hdf5_record_batch): tfrecords_writer.write(example.SerializeToString()) tfrecords_writer.close()
def run(): filename_prefix = "datasets/16x16x16_1-2-3-4-5" filename_template = os.path.join(filename_prefix, file_helpers.DEFAULT_HDF5_TEMPLATE) filename_generator = file_helpers.input_filename_generator_hdf5( filename_template) i = 0 while True: filename = next(filename_generator, None) if filename is None: break # filename_tf = tf.constant(filename, shape=[1]) # filename_queue = tf.train.string_input_producer(filename_tf) # record = data_record.read_and_decode_tf_example(filename_queue) records = data_record.read_hdf5_records_as_list(filename) print(filename) # for record in records: # print(" ", record.action, record.reward) i += 1
def run(args): filename_generator = file_helpers.input_filename_generator_hdf5(args.data_path) filenames = list(filename_generator) np.random.shuffle(filenames) plt.ion() for filename in filenames: print("Filename {}".format(filename)) records = data_record.read_hdf5_records_v4_as_list(filename) indices = np.arange(len(records)) np.random.shuffle(indices) for index in indices: print(" record # {}".format(index)) record = records[index] depth_image = record.depth_image print(np.mean(depth_image)) if depth_image.shape[0] == 1: fig = plt.figure(1) plt.clf() # plt.plot(np.arange(depth_image.shape[1]), depth_image[0, ...]) plt.step(np.arange(depth_image.shape[1]), depth_image[0, ...]) plt.title("Depth image") fig.canvas.draw() plt.show(block=False) else: cv2.imshow("depth_image", depth_image / np.max(depth_image)) cv2.waitKey(50) in_grid_3d = record.in_grid_3d[..., 2:4] fig = 1 fig = plot_grid(in_grid_3d[..., 0], in_grid_3d[..., 1], title_prefix="In ", show=False, fig_offset=fig) out_grid_3d = record.out_grid_3d[..., 2:4] fig = plot_grid(out_grid_3d[..., 0], out_grid_3d[..., 1], title_prefix="Out ", show=False, fig_offset=fig) diff_grid_3d = out_grid_3d - in_grid_3d plot_diff_grid(diff_grid_3d[..., 0], diff_grid_3d[..., 1], title_prefix="Diff ", show=False, fig_offset=fig) print("squared average diff occupancy: {}".format(np.mean(np.square(diff_grid_3d[..., 0])))) print("squared average diff observation count: {}".format(np.mean(np.square(diff_grid_3d[..., 1])))) mlab.show(stop=True)
def run(args): if args.input_path is not None: input_path = args.input_path input_files = list( file_helpers.input_filename_generator_hdf5( input_path, file_helpers.DEFAULT_HDF5_PATTERN)) else: input_list_file = args.input_list_file with file(input_list_file, "r") as fin: input_files = [l.strip() for l in fin.readlines()] dataset_kwargs = {} if args.compression: dataset_kwargs.update({"compression": args.compression}) if args.compression_level >= 0: dataset_kwargs.update({"compression_opts": args.compression_level}) print("Counting {} input files".format(len(input_files))) for i, input_file in enumerate(input_files): print("Reading input file #{} out of {}".format(i, len(input_files))) field_dict = None data, attr_dict = hdf5_utils.read_hdf5_file_to_numpy_dict( input_file, field_dict, read_attributes=True) print("Writing {} samples with new compression settings".format( data[list(data.keys())[0]].shape[0])) hdf5_utils.write_numpy_dict_to_hdf5_file(input_file + "_recompressed", data, attr_dict, **dataset_kwargs) if args.check_written_samples: print("Reading samples from file {}".format(input_file + "_recompressed")) written_data, written_attr_dict = hdf5_utils.read_hdf5_file_to_numpy_dict( input_file + "_recompressed", field_dict, read_attributes=True) for key in data: assert (np.all(data[key] == written_data[key])) for key in attr_dict: assert (np.all(attr_dict[key] == written_attr_dict[key])) os.remove(input_file) os.rename(input_file + "_recompressed", input_file)
def run(args): if args.input_path is not None: input_path = args.input_path input_files = list( file_helpers.input_filename_generator_hdf5( input_path, file_helpers.DEFAULT_HDF5_PATTERN)) else: input_list_file = args.input_list_file with open(input_list_file, "r") as fin: input_files = [l.strip() for l in fin.readlines()] print("Counting {} input files".format(len(input_files))) # Count total number of samples total_num_samples = 0 for i, input_file in enumerate(input_files): print("Reading input file #{} out of {} ({})".format( i, len(input_files), input_file)) field_dict = {"scores": False} data, attr = hdf5_utils.read_hdf5_file_to_numpy_dict( input_file, field_dict, read_attributes=True) total_num_samples += data["scores"].shape[0] print("Total number of samples: {}".format(total_num_samples))
def run(args): if args.input_path is not None: input_path = args.input_path input_files = list( file_helpers.input_filename_generator_hdf5( input_path, file_helpers.DEFAULT_HDF5_PATTERN)) else: input_list_file = args.input_list_file with open(input_list_file, "r") as fin: input_files = [l.strip() for l in fin.readlines()] print("Merging {} input files".format(len(input_files))) output_path1 = args.output_path1 if not os.path.isdir(output_path1): os.makedirs(output_path1) filename_template1 = os.path.join(output_path1, file_helpers.DEFAULT_HDF5_TEMPLATE) output_path2 = args.output_path2 if not os.path.isdir(output_path2): os.makedirs(output_path2) filename_template2 = os.path.join(output_path2, file_helpers.DEFAULT_HDF5_TEMPLATE) samples_per_file = args.samples_per_file split_ratio1 = args.split_ratio1 assert (split_ratio1 > 0) assert (split_ratio1 < 1) dry_run = args.dry_run dataset_kwargs = {} if args.compression: dataset_kwargs.update({"compression": args.compression}) if args.compression_level >= 0: dataset_kwargs.update({"compression_opts": args.compression_level}) # Count total number of samples num_samples = 0 for i, input_file in enumerate(input_files): print("Reading input file #{} out of {}".format(i, len(input_files))) samples = data_record.read_hdf5_records_v4_as_list(input_file) num_samples += len(samples) num_samples1 = round(split_ratio1 * num_samples) num_samples2 = num_samples - num_samples1 if num_samples1 <= 0 or num_samples2 <= 0: import sys sys.stderr.write("Data split will result in empty data set\n") sys.exit(1) def write_samples(samples, next_file_num, filename_template): # filename, next_file_num = get_next_output_tf_filename(next_file_num) filename, next_file_num = file_helpers.get_next_output_hdf5_filename( next_file_num, template=filename_template) print("Writing {} samples to file {}".format(len(samples), filename)) if not dry_run: data_record.write_samples_to_hdf5_file(filename, samples, attr_dict, **dataset_kwargs) if args.check_written_samples: print("Reading samples from file {}".format(filename)) written_samples, written_attr_dict = data_record.read_samples_from_hdf5_file( filename, read_attributes=True) assert (len(samples) == len(written_samples)) for i in range(len(samples)): for key in sample[i]: assert (np.all( samples[i][key] == written_samples[i][key])) for key in attr_dict: assert (np.all(attr_dict[key] == written_attr_dict[key])) return next_file_num # Start with output 1 filename_template = filename_template1 finished_output1 = False written_samples = 0 next_file_num = 0 samples = [] for i, input_file in enumerate(input_files): print("Reading input file #{} out of {}".format(i, len(input_files))) field_dict = None samples, attr_dict = data_record.read_samples_from_hdf5_file( input_file, field_dict) for sample in samples: samples.append(sample) do_write_samples = len(samples) % samples_per_file == 0 if not finished_output1 and written_samples + len( samples) >= num_samples1: do_write_samples = True if do_write_samples: write_samples(samples, next_file_num, filename_template) written_samples += len(samples) samples = [] if not finished_output1 and written_samples >= num_samples1: finished_output1 = True filename_template = filename_template2 next_file_num = 0 if len(samples) > 0: write_samples(samples, next_file_num, filename_template) samples = [] assert (len(samples) == 0)
def write_hdf5_files_to_lmdb(input_and_target_retriever, tf_cfg, hdf5_path, lmdb_path, max_num_files, batch_size, serialization_name, compression, compression_arg, append=False, max_num_samples=None, verbose=False): logger.info("Data path: {}".format(hdf5_path)) filename_generator = file_helpers.input_filename_generator_hdf5(hdf5_path) filenames = list(filename_generator) # Write filenames to lmdb storage path (for later reference) if append: file_mode = "a" else: file_mode = "w" with open(lmdb_path + "_filenames", file_mode) as fout: for filename in filenames: fout.write("{}\n".format(os.path.basename(filename))) if len(filenames) == 0: raise RuntimeError("No dataset file") else: logger.info("Found {} dataset files".format(len(filenames))) # Limit dataset size? if max_num_files > 0: filenames = filenames[:max_num_files] logger.info("Using {} dataset files".format(len(filenames))) # Force recomputation of num of samples if not using all data files else: logger.info("Using all dataset files") if verbose: # Retrieve input and target shapes tmp_data = input_and_target_retriever.read_data_from_file(filenames[0]) input_shape = input_and_target_retriever.get_input_from_data( tmp_data).shape[1:] target_shape = input_and_target_retriever.get_target_from_data( tmp_data).shape[1:] del tmp_data logger.info("Input and target shapes:") logger.info(" Shape of input: {}".format(input_shape)) logger.info(" Shape of target: {}".format(target_shape)) if verbose: input_pipeline.print_data_stats(filenames[0], input_and_target_retriever) coord = tf.train.Coordinator() # Create HDF5 readers hdf5_pipeline = hdf5_utils.HDF5ReaderProcessCoordinator( filenames, coord, read_data_fn=input_and_target_retriever.read_data_from_file, shuffle=False, repeats=1, num_processes=tf_cfg.cpu_queue_processes, data_queue_capacity=tf_cfg.cpu_data_queue_capacity, verbose=verbose) try: custom_threads = [] hdf5_pipeline.start() custom_threads.extend(hdf5_pipeline.threads) class HDF5DataFlow(tensorpack.dataflow.DataFlow): def __init__(self, hdf5_pipeline, max_num_samples=None): self._hdf5_pipeline = hdf5_pipeline self._num_samples = 0 self._max_num_samples = max_num_samples def get_data(self): input_field_name = input_and_target_retriever.input_field_name target_field_name = input_and_target_retriever.target_field_name while True: if verbose: logger.debug( "Fetching next data block from hdf5 pipeline") full_data = self._hdf5_pipeline.get_next_data() if full_data == hdf5_utils.QUEUE_END: return data = { input_field_name: full_data[input_field_name], target_field_name: full_data[target_field_name], } # TODO: Remove this. Kind of a hack. data = { key: np.asarray(array, dtype=np.float32) for key, array in data.items() } #self._num_samples += data[input_field_name].shape[0] if verbose: logger.debug("Iterating through data block") for i in range(data[input_field_name].shape[0]): sample = [{ key: batch[i, ...] for key, batch in data.items() }] yield sample self._num_samples += 1 if self._max_num_samples is not None and self._max_num_samples > 0: if self._num_samples >= self._max_num_samples: return @property def num_samples(self): return self._num_samples if append: logger.info( "Appending HDF5 data from {} to lmdb database {}".format( hdf5_path, lmdb_path)) lmdb_df = tensorpack_utils.AutoLMDBData(lmdb_path, shuffle=False) initial_num_samples = lmdb_df.size() logger.info("initial_num_samples: {}".format(initial_num_samples)) else: logger.info("Writing HDF5 data from {} to lmdb database {}".format( hdf5_path, lmdb_path)) initial_num_samples = 0 if max_num_samples is not None and max_num_samples > 0: logger.info( "Limiting num of samples to {}".format(max_num_samples)) max_num_samples = max_num_samples - initial_num_samples if max_num_samples < 0: logger.info("WARNING: Database already has enough samples") return if append: logger.info( "Appending at most {} samples".format(max_num_samples)) hdf5_df = HDF5DataFlow(hdf5_pipeline, max_num_samples) tensorpack_utils.dump_compressed_dataflow_to_lmdb( hdf5_df, lmdb_path, batch_size, write_frequency=10, serialization_name=serialization_name, compression=compression, compression_arg=compression_arg, append=append) if batch_size > 0: num_dropped_samples = hdf5_df.num_samples % batch_size else: num_dropped_samples = 0 if batch_size > 0: lmdb_df = tensorpack_utils.LMDBDataWithMetaData(lmdb_path, shuffle=False) logger.info("Database has {} batches".format(lmdb_df.size())) logger.info("hdf5_df.num_samples: {}".format(hdf5_df.num_samples)) logger.info("num_dropped_samples: {}".format(num_dropped_samples)) logger.info("batch_size: {}".format(batch_size)) logger.info("lmdb_df.size(): {}".format(lmdb_df.size())) if initial_num_samples + hdf5_df.num_samples - num_dropped_samples != batch_size * lmdb_df.size( ): logger.info( "initial_num_samples: {}".format(initial_num_samples)) logger.info("hdf5_df.num_samples: {}".format( hdf5_df.num_samples)) logger.info( "num_dropped_samples: {}".format(num_dropped_samples)) logger.info("batch_size: {}".format(batch_size)) logger.info("lmdb_df.size(): {}".format(lmdb_df.size())) assert (initial_num_samples + hdf5_df.num_samples - num_dropped_samples == batch_size * lmdb_df.size()) lmdb_df = tensorpack_utils.AutoLMDBData(lmdb_path, shuffle=False) logger.info("Database has {} samples".format(lmdb_df.size())) logger.info("hdf5_df.num_samples: {}".format(hdf5_df.num_samples)) logger.info("num_dropped_samples: {}".format(num_dropped_samples)) logger.info("batch_size: {}".format(batch_size)) logger.info("lmdb_df.size(): {}".format(lmdb_df.size())) assert (initial_num_samples + hdf5_df.num_samples - num_dropped_samples == lmdb_df.size()) # Check that we can read data without errors lmdb_df.reset_state() it = lmdb_df.get_data() q = next(it) for key in q[0]: logger.info(q[0][key].shape) logger.info(q[0][key].dtype) assert (q[0][key].dtype == np.float32) if num_dropped_samples > 0: logger.warn("Dropped {} samples from input dataset".format( num_dropped_samples)) hdf5_pipeline.stop() except Exception as exc: logger.info( "Exception while converting hdf5 data to LMDB database: {}".format( exc)) traceback.print_exc() coord.request_stop(exc) raise exc finally: logger.info("Requesting stop") coord.request_stop() coord.join(custom_threads, stop_grace_period_secs=10)
def run(args): if args.input_path is not None: input_path = args.input_path input_files = list( file_helpers.input_filename_generator_hdf5( input_path, file_helpers.DEFAULT_HDF5_PATTERN)) else: input_list_file = args.input_list_file with file(input_list_file, "r") as fin: input_files = [l.strip() for l in fin.readlines()] print("Merging {} input files".format(len(input_files))) output_path = args.output_path if not os.path.isdir(output_path): os.makedirs(output_path) filename_template = os.path.join(output_path, file_helpers.DEFAULT_HDF5_TEMPLATE) samples_per_file = args.samples_per_file check_written_samples = args.check_written_samples dry_run = args.dry_run dataset_kwargs = {} if args.compression: dataset_kwargs.update({"compression": args.compression}) if args.compression_level >= 0: dataset_kwargs.update({"compression_opts": args.compression_level}) def write_samples(samples, next_file_num): # filename, next_file_num = get_next_output_tf_filename(next_file_num) filename, next_file_num = file_helpers.get_next_output_hdf5_filename( next_file_num, template=filename_template) print("Writing {} samples to file {}".format(len(samples), filename)) if not dry_run: data_record.write_samples_to_hdf5_file(filename, samples, attr_dict, **dataset_kwargs) if args.check_written_samples: print("Reading samples from file {}".format(filename)) written_samples, written_attr_dict = data_record.read_samples_from_hdf5_file( filename, read_attributes=True) assert (len(samples) == len(written_samples)) for i in range(len(samples)): for key in sample[i]: assert (np.all( samples[i][key] == written_samples[i][key])) for key in attr_dict: assert (np.all(attr_dict[key] == written_attr_dict[key])) return next_file_num total_num_samples = 0 next_file_num = 0 samples = [] for i, input_file in enumerate(input_files): print("Reading input file #{} out of {}".format(i, len(input_files))) field_dict = None samples, attr_dict = data_record.read_samples_from_hdf5_file( input_file, field_dict) for sample in samples: samples.append(sample) if len(samples) % samples_per_file == 0: next_file_num = write_samples(samples, next_file_num) samples = [] if len(samples) > 0: write_samples(samples, next_file_num) samples = [] assert (len(samples) == 0) print("Total number of written samples: {}".format(total_num_samples))