def __init__(self, filename, config, access_mode=WRITE_ACCESS): # read params self._filename = filename self._config = config self._datapoints_per_file = config['datapoints_per_file'] self._access_mode = access_mode # check valid access mode if access_mode == READ_WRITE_ACCESS: raise ValueError('Read and write not supported simultaneously.') # open dataset folder # create dataset if necessary if not os.path.exists( self._filename) and access_mode != READ_ONLY_ACCESS: os.mkdir(self._filename) # throw error if dataset doesn't exist elif not os.path.exists( self._filename) and access_mode == READ_ONLY_ACCESS: raise ValueError('Dataset %s does not exist!' % (self._filename)) # check dataset empty elif os.path.exists(self._filename) and len(os.listdir( self._filename)) > 0 and access_mode == WRITE_ACCESS: human_input = utils.keyboard_input( 'Dataset %s exists. Overwrite?' % (self.filename), yesno=True) if human_input.lower() == 'n': raise ValueError('User opted not to overwrite dataset') # save config to location if access_mode != READ_ONLY_ACCESS: config_filename = os.path.join(self._filename, 'config.json') json.dump(self._config, open(config_filename, 'w')) # init data storage self._allocate_tensors() # init state variables if access_mode == WRITE_ACCESS: # init no files self._num_tensors = 0 self._num_datapoints = 0 if not os.path.exists(self.tensor_dir): os.mkdir(self.tensor_dir) elif access_mode == READ_ONLY_ACCESS: # read the number of tensor files tensor_dir = self.tensor_dir tensor_filenames = utils.filenames(tensor_dir, tag=COMPRESSED_TENSOR_EXT, sorted=True) file_nums = np.array( [int(filename[-9:-4]) for filename in tensor_filenames]) self._num_tensors = np.max(file_nums) + 1 # compute the number of datapoints last_tensor_ind = np.where(file_nums == self._num_tensors - 1)[0][0] last_tensor_data = np.load( tensor_filenames[last_tensor_ind])['arr_0'] self._num_datapoints_last_file = last_tensor_data.shape[0] self._num_datapoints = self._datapoints_per_file * ( self._num_tensors - 1) + self._num_datapoints_last_file # form index maps for each file self._index_to_file_num = {} self._file_num_to_indices = {} # set file index cur_file_num = 0 start_datapoint_index = 0 # set mapping from file num to datapoint indices self._file_num_to_indices[cur_file_num] = np.arange( self._datapoints_per_file) + start_datapoint_index for ind in range(self._num_datapoints): # set mapping from index to file num self._index_to_file_num[ind] = cur_file_num # update to the next file if ind > 0 and ind % self._datapoints_per_file == 0: cur_file_num += 1 start_datapoint_index += self._datapoints_per_file # set mapping from file num to datapoint indices if cur_file_num < self._num_tensors - 1: self._file_num_to_indices[cur_file_num] = np.arange( self._datapoints_per_file) + start_datapoint_index else: self._file_num_to_indices[cur_file_num] = np.arange( self._num_datapoints_last_file ) + start_datapoint_index else: raise ValueError('Access mode %s not supported' % (access_mode))
from autolab_core.constants import * import autolab_core.utils as utils if __name__ == '__main__': # initialize logging logging.getLogger().setLevel(logging.INFO) # parse args parser = argparse.ArgumentParser(description='Convert a legacy training dataset to TensorDataset (saves in-place)') parser.add_argument('dataset_dir', type=str, default=None, help='path to a tensor dataset') args = parser.parse_args() dataset_dir = args.dataset_dir # read filenames filenames = utils.filenames(dataset_dir) # create config file datapoints_per_file = None field_config = {} for filename in filenames: _, f = os.path.split(filename) _, ext = os.path.splitext(f) if ext != '.npz': continue u_ind = f.rfind('_') field_name = f[:u_ind] if field_name not in field_config.keys(): field_config[field_name] = {}
config_filename = args.config_filename # open config file cfg = YamlConfig(config_filename) input_dataset_names = cfg["input_datasets"] output_dataset_name = cfg["output_dataset"] display_rate = cfg["display_rate"] # modify list of dataset names all_input_dataset_names = [] for dataset_name in input_dataset_names: tensor_dir = os.path.join(dataset_name, "tensors") if os.path.exists(tensor_dir): all_input_dataset_names.append(dataset_name) else: dataset_subdirs = utils.filenames(dataset_name, tag="dataset_") all_input_dataset_names.extend(dataset_subdirs) # open tensor dataset dataset = TensorDataset.open(all_input_dataset_names[0]) tensor_config = copy.deepcopy(dataset.config) for field_name in cfg["exclude_fields"]: if field_name in tensor_config["fields"].keys(): del tensor_config["fields"][field_name] field_names = tensor_config["fields"].keys() alt_field_names = [ f if f != "rewards" else "grasp_metrics" for f in field_names ] # init tensor dataset output_dataset = TensorDataset(output_dataset_name, tensor_config)
""" Lists the set of watertight meshes from an input directory. Author: Jeff Mahler """ import numpy as np import os import sys import autolab_core.utils as utils import trimesh if __name__ == '__main__': data_dir = sys.argv[1] out_filename = os.path.join(data_dir, 'watertight_listing.txt') out_f = open(out_filename, 'w') num_watertight = 0 obj_filenames = utils.filenames(data_dir, tag='.obj') for k, obj_filename in enumerate(obj_filenames): mesh = trimesh.load_mesh(obj_filename) if mesh.is_watertight: num_watertight += 1 print 'Pct Watertight:', float(num_watertight) / (k + 1) print 'Pct Complete:', float(k) / len(obj_filenames) print obj_filename out_f.write('%s\n' % (obj_filename)) print 'Pct Watertight:', float(num_watertight) / len(obj_filenames) out_f.close()
type=str, help='directory containing 3D model files to convert') parser.add_argument('--output_dir', type=str, default=None, help='directory to save .OBJ files to') args = parser.parse_args() data_dir = args.input_dir output_dir = args.output_dir if output_dir is None: output_dir = data_dir # get model filenames model_filenames = [] for ext in SUPPORTED_EXTENSIONS: model_filenames.extend(utils.filenames(data_dir, tag=ext)) model_file_roots = [] for model_filename in model_filenames: root, _ = os.path.splitext(model_filename) model_file_roots.append(root) # create obj filenames obj_filenames = [f + '.obj' for f in model_file_roots] obj_filenames = [f.replace(data_dir, output_dir) for f in obj_filenames] num_files = len(obj_filenames) # convert using meshlab server i = 0 for obj_filename, model_filename in zip(obj_filenames, model_filenames): logging.info('Converting %s (%d of %d)' % (model_filename, i + 1, num_files))
config_filename = args.config_filename # open config file cfg = YamlConfig(config_filename) input_dataset_names = cfg['input_datasets'] output_dataset_name = cfg['output_dataset'] display_rate = cfg['display_rate'] # modify list of dataset names all_input_dataset_names = [] for dataset_name in input_dataset_names: tensor_dir = os.path.join(dataset_name, 'tensors') if os.path.exists(tensor_dir): all_input_dataset_names.append(dataset_name) else: dataset_subdirs = utils.filenames(dataset_name, tag='dataset_') all_input_dataset_names.extend(dataset_subdirs) # open tensor dataset dataset = TensorDataset.open(all_input_dataset_names[0]) tensor_config = copy.deepcopy(dataset.config) for field_name in cfg['exclude_fields']: if field_name in tensor_config['fields'].keys(): del tensor_config['fields'][field_name] field_names = tensor_config['fields'].keys() alt_field_names = [ f if f != 'rewards' else 'grasp_metrics' for f in field_names ] # init tensor dataset output_dataset = TensorDataset(output_dataset_name, tensor_config)
type=str, default=None, help='directory containing the models to compare') parser.add_argument('--config_filename', type=str, default='cfg/compare_models.yaml', help='configuration file to use') args = parser.parse_args() model_dir = args.model_dir config_filename = args.config_filename # read config config = YamlConfig(config_filename) # read all models model_subdirs = utils.filenames(model_dir) # remove models outside of the time scope new_model_subdirs = [] for model_subdir in model_subdirs: stats = os.stat(model_subdir) model_path, model_ext = os.path.splitext(model_subdir) if model_ext != '': continue model_path, model_root = os.path.split(model_subdir) if model_root in IGNORE_MODELS: continue dt = datetime.datetime.fromtimestamp(stats.st_mtime) if dt.year >= config['earliest']['year'] and \ dt.month >= config['earliest']['month'] and \ dt.day >= config['earliest']['day']: