def __init__(self, filename, config, access_mode=WRITE_ACCESS):
# read params
self._filename = filename
self._config = config
self._datapoints_per_file = config['datapoints_per_file']
self._access_mode = access_mode
# check valid access mode
if access_mode == READ_WRITE_ACCESS:
raise ValueError('Read and write not supported simultaneously.')
# open dataset folder
# create dataset if necessary
if not os.path.exists(
self._filename) and access_mode != READ_ONLY_ACCESS:
os.mkdir(self._filename)
# throw error if dataset doesn't exist
elif not os.path.exists(
self._filename) and access_mode == READ_ONLY_ACCESS:
raise ValueError('Dataset %s does not exist!' % (self._filename))
# check dataset empty
elif os.path.exists(self._filename) and len(os.listdir(
self._filename)) > 0 and access_mode == WRITE_ACCESS:
human_input = utils.keyboard_input(
'Dataset %s exists. Overwrite?' % (self.filename), yesno=True)
if human_input.lower() == 'n':
raise ValueError('User opted not to overwrite dataset')
# save config to location
if access_mode != READ_ONLY_ACCESS:
config_filename = os.path.join(self._filename, 'config.json')
json.dump(self._config, open(config_filename, 'w'))
# init data storage
self._allocate_tensors()
# init state variables
if access_mode == WRITE_ACCESS:
# init no files
self._num_tensors = 0
self._num_datapoints = 0
if not os.path.exists(self.tensor_dir):
os.mkdir(self.tensor_dir)
elif access_mode == READ_ONLY_ACCESS:
# read the number of tensor files
tensor_dir = self.tensor_dir
tensor_filenames = utils.filenames(tensor_dir,
tag=COMPRESSED_TENSOR_EXT,
sorted=True)
file_nums = np.array(
[int(filename[-9:-4]) for filename in tensor_filenames])
self._num_tensors = np.max(file_nums) + 1
# compute the number of datapoints
last_tensor_ind = np.where(file_nums == self._num_tensors -
1)[0][0]
last_tensor_data = np.load(
tensor_filenames[last_tensor_ind])['arr_0']
self._num_datapoints_last_file = last_tensor_data.shape[0]
self._num_datapoints = self._datapoints_per_file * (
self._num_tensors - 1) + self._num_datapoints_last_file
# form index maps for each file
self._index_to_file_num = {}
self._file_num_to_indices = {}
# set file index
cur_file_num = 0
start_datapoint_index = 0
# set mapping from file num to datapoint indices
self._file_num_to_indices[cur_file_num] = np.arange(
self._datapoints_per_file) + start_datapoint_index
for ind in range(self._num_datapoints):
# set mapping from index to file num
self._index_to_file_num[ind] = cur_file_num
# update to the next file
if ind > 0 and ind % self._datapoints_per_file == 0:
cur_file_num += 1
start_datapoint_index += self._datapoints_per_file
# set mapping from file num to datapoint indices
if cur_file_num < self._num_tensors - 1:
self._file_num_to_indices[cur_file_num] = np.arange(
self._datapoints_per_file) + start_datapoint_index
else:
self._file_num_to_indices[cur_file_num] = np.arange(
self._num_datapoints_last_file
) + start_datapoint_index
else:
raise ValueError('Access mode %s not supported' % (access_mode))
from autolab_core.constants import *
import autolab_core.utils as utils
if __name__ == '__main__':
# initialize logging
logging.getLogger().setLevel(logging.INFO)
# parse args
parser = argparse.ArgumentParser(description='Convert a legacy training dataset to TensorDataset (saves in-place)')
parser.add_argument('dataset_dir', type=str, default=None, help='path to a tensor dataset')
args = parser.parse_args()
dataset_dir = args.dataset_dir
# read filenames
filenames = utils.filenames(dataset_dir)
# create config file
datapoints_per_file = None
field_config = {}
for filename in filenames:
_, f = os.path.split(filename)
_, ext = os.path.splitext(f)
if ext != '.npz':
continue
u_ind = f.rfind('_')
field_name = f[:u_ind]
if field_name not in field_config.keys():
field_config[field_name] = {}
config_filename = args.config_filename
# open config file
cfg = YamlConfig(config_filename)
input_dataset_names = cfg["input_datasets"]
output_dataset_name = cfg["output_dataset"]
display_rate = cfg["display_rate"]
# modify list of dataset names
all_input_dataset_names = []
for dataset_name in input_dataset_names:
tensor_dir = os.path.join(dataset_name, "tensors")
if os.path.exists(tensor_dir):
all_input_dataset_names.append(dataset_name)
else:
dataset_subdirs = utils.filenames(dataset_name, tag="dataset_")
all_input_dataset_names.extend(dataset_subdirs)
# open tensor dataset
dataset = TensorDataset.open(all_input_dataset_names[0])
tensor_config = copy.deepcopy(dataset.config)
for field_name in cfg["exclude_fields"]:
if field_name in tensor_config["fields"].keys():
del tensor_config["fields"][field_name]
field_names = tensor_config["fields"].keys()
alt_field_names = [
f if f != "rewards" else "grasp_metrics" for f in field_names
]
# init tensor dataset
output_dataset = TensorDataset(output_dataset_name, tensor_config)
"""
Lists the set of watertight meshes from an input directory.
Author: Jeff Mahler
"""
import numpy as np
import os
import sys
import autolab_core.utils as utils
import trimesh
if __name__ == '__main__':
data_dir = sys.argv[1]
out_filename = os.path.join(data_dir, 'watertight_listing.txt')
out_f = open(out_filename, 'w')
num_watertight = 0
obj_filenames = utils.filenames(data_dir, tag='.obj')
for k, obj_filename in enumerate(obj_filenames):
mesh = trimesh.load_mesh(obj_filename)
if mesh.is_watertight:
num_watertight += 1
print 'Pct Watertight:', float(num_watertight) / (k + 1)
print 'Pct Complete:', float(k) / len(obj_filenames)
print obj_filename
out_f.write('%s\n' % (obj_filename))
print 'Pct Watertight:', float(num_watertight) / len(obj_filenames)
out_f.close()
type=str,
help='directory containing 3D model files to convert')
parser.add_argument('--output_dir',
type=str,
default=None,
help='directory to save .OBJ files to')
args = parser.parse_args()
data_dir = args.input_dir
output_dir = args.output_dir
if output_dir is None:
output_dir = data_dir
# get model filenames
model_filenames = []
for ext in SUPPORTED_EXTENSIONS:
model_filenames.extend(utils.filenames(data_dir, tag=ext))
model_file_roots = []
for model_filename in model_filenames:
root, _ = os.path.splitext(model_filename)
model_file_roots.append(root)
# create obj filenames
obj_filenames = [f + '.obj' for f in model_file_roots]
obj_filenames = [f.replace(data_dir, output_dir) for f in obj_filenames]
num_files = len(obj_filenames)
# convert using meshlab server
i = 0
for obj_filename, model_filename in zip(obj_filenames, model_filenames):
logging.info('Converting %s (%d of %d)' %
(model_filename, i + 1, num_files))
config_filename = args.config_filename
# open config file
cfg = YamlConfig(config_filename)
input_dataset_names = cfg['input_datasets']
output_dataset_name = cfg['output_dataset']
display_rate = cfg['display_rate']
# modify list of dataset names
all_input_dataset_names = []
for dataset_name in input_dataset_names:
tensor_dir = os.path.join(dataset_name, 'tensors')
if os.path.exists(tensor_dir):
all_input_dataset_names.append(dataset_name)
else:
dataset_subdirs = utils.filenames(dataset_name, tag='dataset_')
all_input_dataset_names.extend(dataset_subdirs)
# open tensor dataset
dataset = TensorDataset.open(all_input_dataset_names[0])
tensor_config = copy.deepcopy(dataset.config)
for field_name in cfg['exclude_fields']:
if field_name in tensor_config['fields'].keys():
del tensor_config['fields'][field_name]
field_names = tensor_config['fields'].keys()
alt_field_names = [
f if f != 'rewards' else 'grasp_metrics' for f in field_names
]
# init tensor dataset
output_dataset = TensorDataset(output_dataset_name, tensor_config)
type=str,
default=None,
help='directory containing the models to compare')
parser.add_argument('--config_filename',
type=str,
default='cfg/compare_models.yaml',
help='configuration file to use')
args = parser.parse_args()
model_dir = args.model_dir
config_filename = args.config_filename
# read config
config = YamlConfig(config_filename)
# read all models
model_subdirs = utils.filenames(model_dir)
# remove models outside of the time scope
new_model_subdirs = []
for model_subdir in model_subdirs:
stats = os.stat(model_subdir)
model_path, model_ext = os.path.splitext(model_subdir)
if model_ext != '':
continue
model_path, model_root = os.path.split(model_subdir)
if model_root in IGNORE_MODELS:
continue
dt = datetime.datetime.fromtimestamp(stats.st_mtime)
if dt.year >= config['earliest']['year'] and \
dt.month >= config['earliest']['month'] and \
dt.day >= config['earliest']['day']:
