def merge_all_classes_to_dataset(csv_files_list, balance = True): count_direction = [] item_data_list = {} item_data_selected = {} for i in range(len(csv_files_list)): csv_file_path = csv_files_list[i] temp_data = c_io.read_csv(csv_file_path) items_data = temp_data[1:] # ignore the first row with labels count_direction.append(len(items_data)) item_data_list[csv_files_list[i]] = items_data if balance: min_count = min(count_direction) print("old min", min_count) min_count = min(min_count, 100000) print("capped min", min_count) number_per_class = [min(x, min_count) for x in count_direction] else: number_per_class = count_direction print("number for each class ",number_per_class ) for i in range(len(csv_files_list)): indices = lt.shuffle_indices(count_direction[i], shuffle = True) indices = indices[:number_per_class[i]] items_selected = lt.get_list_items_with_indices(item_data_list[csv_files_list[i]], indices) item_data_selected[csv_files_list[i]] = items_selected return item_data_selected
def merge_all_scenes_to_classes(src_scenes): items_for_all_scenes_list = [] csv_file_path_list = [] direction_count = [] for i in range(len(label_list)): items_for_all_scenes = [] items_for_all_scenes_list.append(items_for_all_scenes) for scene in src_scenes: for i in range(len(label_list)): csv_file_path = os.path.join( dst_dataset_folder, "{}_{}.csv".format(scene, label_list[i])) if os.path.exists(csv_file_path): temp_data = c_io.read_csv(csv_file_path) items_for_all_scenes_list[ i] = items_for_all_scenes_list[i] + temp_data[ 1:] # concatenate two lists together, each list remove the colume names for i in range(len(label_list)): csv_file_path = os.path.join(dst_dataset_folder, "{}.csv".format(label_list[i])) csv_file_path_list.append(csv_file_path) direction_count.append(len(items_for_all_scenes_list[i])) csv_lines = prepare_csv_data(items_for_all_scenes_list[i]) c_io.save_csv(csv_file_path, csv_lines) return csv_file_path_list, direction_count
def __init__(self, **kwargs): h5_file = kwargs.pop('h5_file') csv_file = kwargs.pop('csv_file') transform_ = kwargs.pop('transform') option = kwargs.pop('option') self.data_h5 = h5_file self.gt_info = c_io.read_csv(csv_file) if option == "2DScaled" or option == "CuboidTwoStep2DScaled": print("Reading data from {}".format(h5_file)) self.dataset = cuboidTwoStep2DScaledDatasetH5( self.data_h5, self.gt_info, transform_) elif option == "2D" or option == "CuboidTwoStep2D": self.dataset = cuboidTwoStep2DatasetH5(self.data_h5, self.gt_info, transform_) elif option == "5D" or option == "CuboidTwoStep5D": self.dataset = cuboidTwoStep5DatasetH5(self.data_h5, self.gt_info, transform_) elif option == "4D" or option == "CuboidTwoStep4D": self.dataset = cuboidTwoStep4DatasetH5(self.data_h5, self.gt_info, transform_) elif option == "UtilityOnly" or option == "CuboidTwoStepUtilityOnly": self.dataset = cuboidTwoStepUtilityOnlyDatasetH5( self.data_h5, self.gt_info, transform_) super(TwoStepH5Loader, self).__init__(self.dataset, **kwargs)
def __init__(self, csv_file, size, transform=None): """ Args: csv_file (string): Path to the csv file with annotations. transform (callable, optional): Optional transform to be applied on a sample. """ self.gt_info = c_io.read_csv(csv_file) self.transform = transform self.size = size
def __init__(self, csv_file, h, w, transform=None): """ Args: csv_file (string): Path to the csv file with annotations. transform (callable, optional): Optional transform to be applied on a sample. """ self.gt_info = c_io.read_csv(csv_file) self.transform = transform self.height = h self.width = w self.label_list = ["up", "down", "left", "right"]