def setup_test_data(self, test_data_config: Optional[DictConfig] = None): if test_data_config is None: test_data_config = self._cfg.test_ds labels_file = os.path.join(self._cfg.dataset.data_dir, test_data_config.labels_file) get_label_ids( label_file=labels_file, is_training=False, pad_label=self._cfg.dataset.pad_label, label_ids_dict=self._cfg.label_ids, get_weights=False, ) self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config)
def setup_training_data(self, train_data_config: Optional[DictConfig] = None): if train_data_config is None: train_data_config = self._cfg.train_ds labels_file = os.path.join(self._cfg.dataset.data_dir, train_data_config.labels_file) # for older(pre - 1.0.0.b3) configs compatibility if not hasattr(self._cfg, "class_labels") or self._cfg.class_labels is None: OmegaConf.set_struct(self._cfg, False) self._cfg.class_labels = {} self._cfg.class_labels = OmegaConf.create( {'class_labels_file': 'label_ids.csv'}) OmegaConf.set_struct(self._cfg, True) label_ids, label_ids_filename, self.class_weights = get_label_ids( label_file=labels_file, is_training=True, pad_label=self._cfg.dataset.pad_label, label_ids_dict=self._cfg.label_ids, get_weights=True, class_labels_file_artifact=self._cfg.class_labels. class_labels_file, ) # save label maps to the config self._cfg.label_ids = OmegaConf.create(label_ids) self.register_artifact('class_labels.class_labels_file', label_ids_filename) self._train_dl = self._setup_dataloader_from_config( cfg=train_data_config)
def setup_training_data(self, train_data_config: Optional[DictConfig] = None): if train_data_config is None: train_data_config = self._cfg.train_ds labels_file = os.path.join(self._cfg.dataset.data_dir, train_data_config.labels_file) label_ids, label_ids_filename, self.class_weights = get_label_ids( label_file=labels_file, is_training=True, pad_label=self._cfg.dataset.pad_label, label_ids_dict=self._cfg.label_ids, get_weights=self._cfg.dataset.class_balancing == 'weighted_loss', ) # save label maps to the config self._cfg.label_ids = OmegaConf.create(label_ids) self.register_artifact('label_ids.csv', label_ids_filename) self._train_dl = self._setup_dataloader_from_config( cfg=train_data_config)