def read_config(self, config_params, datasets_index, **kwargs): """ Read the config file and the datasets index Between the config file and the dataset index, we have enough information to configure the backend and the models. We can also initialize the data readers :param config_file: The config file :param datasets_index: The index of datasets :return: """ datasets_index = read_config_file_or_json(datasets_index, 'datasets') datasets_set = index_by_label(datasets_index) self.config_params = config_params basedir = self.get_basedir() if basedir is not None and not os.path.exists(basedir): logger.info('Creating: {}'.format(basedir)) os.mkdir(basedir) self.config_params['train']['basedir'] = basedir # Read GPUS from env variables now so that the reader has access if self.config_params['model'].get('gpus', 1) == -1: self.config_params['model']['gpus'] = len(get_env_gpus()) self.config_file = kwargs.get('config_file') self._setup_task(**kwargs) self._load_user_modules() self._configure_reporting(config_params.get('reporting', {}), **kwargs) self.dataset = datasets_set[self.config_params['dataset']] self.reader = self._create_task_specific_reader()
def read_config(self, config_params, datasets_index, **kwargs): """ Read the config file and the datasets index Between the config file and the dataset index, we have enough information to configure the backend and the models. We can also initialize the data readers :param config_file: The config file :param datasets_index: The index of datasets :return: """ datasets_index = read_config_file_or_json(datasets_index, 'datasets') datasets_set = index_by_label(datasets_index) self.config_params = config_params config_file = deepcopy(config_params) basedir = self.get_basedir() if basedir is not None and not os.path.exists(basedir): logger.info('Creating: %s', basedir) os.makedirs(basedir) self.config_params['train']['basedir'] = basedir # Read GPUS from env variables now so that the reader has access if self.config_params['model'].get('gpus', -1) == -1: self.config_params['model']['gpus'] = len(get_env_gpus()) self._setup_task(**kwargs) self._load_user_modules() self.dataset = get_dataset_from_key(self.config_params['dataset'], datasets_set) # replace dataset in config file by the latest dataset label, this will be used by some reporting hooks config_file['dataset'] = self.dataset['label'] self._configure_reporting(config_params.get('reporting', {}), config_file=config_file, **kwargs) self.reader = self._create_task_specific_reader()
def read_config(self, config_params, datasets_index, **kwargs): """ Read the config file and the datasets index Between the config file and the dataset index, we have enough information to configure the backend and the models. We can also initialize the data readers :param config_file: The config file :param datasets_index: The index of datasets :return: """ datasets_index = read_config_file_or_json(datasets_index, 'datasets') datasets_set = index_by_label(datasets_index) self.config_params = config_params basedir = self.get_basedir() if basedir is not None and not os.path.exists(basedir): logger.info('Creating: %s', basedir) os.makedirs(basedir) self.config_params['train']['basedir'] = basedir # Read GPUS from env variables now so that the reader has access if self.config_params['model'].get('gpus', -1) == -1: self.config_params['model']['gpus'] = len(get_env_gpus()) self.config_file = kwargs.get('config_file') self._setup_task(**kwargs) self._load_user_modules() self._configure_reporting(config_params.get('reporting', {}), **kwargs) self.dataset = get_dataset_from_key(self.config_params['dataset'], datasets_set) self.reader = self._create_task_specific_reader()
def test_none(remove_envs): gold = ['0'] gpus = get_env_gpus() assert gpus == gold
def test_visible_first(cuda_visible, nv_gpu): gpus = get_env_gpus() assert gpus != nv_gpu assert gpus == cuda_visible
def test_visible(cuda_visible): gpus = get_env_gpus() assert gpus == cuda_visible
def test_none(remove_envs): gold = ['0'] gpus = get_env_gpus() assert gpus == gold
def test_visible_first(cuda_visible, nv_gpu): gpus = get_env_gpus() assert gpus != nv_gpu assert gpus == cuda_visible
def test_visible(cuda_visible): gpus = get_env_gpus() assert gpus == cuda_visible