def score(self, test_data: DLDataset, mode='test', metric_func=None): if metric_func is None: metric_func = self.metric preds = self.predict(test_data, mode=mode) labels = test_data.get_labels(mode=mode) # TODO: support AUC return metric_func._score_func(preds, labels)
def load_predict_data(self, test_data: DLDataset): if self.task_type == IMG_CLS: test_transforms = get_test_transforms(self.best_algo_config, image_size=self.image_size) test_data.load_test_data(test_transforms) test_data.load_data(test_transforms, test_transforms) else: test_data.load_test_data() test_data.load_data()
def predict(self, test_data: DLDataset, mode='test'): model_pred_list = list() final_pred = list() model_cnt = 0 for algo_id in self.stats["include_algorithms"]: model_configs = self.stats[algo_id]['model_configs'] for idx, config in enumerate(model_configs): if self.task_type == IMG_CLS: test_transforms = get_test_transforms(config, image_size=self.image_size) test_data.load_test_data(test_transforms) test_data.load_data(test_transforms, test_transforms) else: test_data.load_test_data() test_data.load_data() if mode == 'test': dataset = test_data.test_dataset else: if test_data.subset_sampler_used: dataset = test_data.train_dataset else: dataset = test_data.val_dataset estimator = get_estimator_with_parameters(self.task_type, config, self.max_epoch, dataset, self.timestamp, device=self.device) if self.task_type in CLS_TASKS: if mode == 'test': model_pred_list.append(estimator.predict_proba(test_data.test_dataset)) else: if test_data.subset_sampler_used: model_pred_list.append( estimator.predict_proba(test_data.train_dataset, sampler=test_data.val_sampler)) else: model_pred_list.append(estimator.predict_proba(test_data.val_dataset)) else: if mode == 'test': model_pred_list.append(estimator.predict(test_data.test_dataset)) else: if test_data.subset_sampler_used: model_pred_list.append( estimator.predict(test_data.train_dataset, sampler=test_data.val_sampler)) else: model_pred_list.append(estimator.predict(test_data.val_dataset)) model_cnt += 1 # Calculate the average of predictions for i in range(len(model_pred_list[0])): sample_pred_list = [model_pred[i] for model_pred in model_pred_list] pred_average = reduce(lambda x, y: x + y, sample_pred_list) / len(sample_pred_list) final_pred.append(pred_average) return np.array(final_pred)
def refit(self, dataset: DLDataset): if self.es is None: config_dict = self.best_algo_config.get_dictionary().copy() # TODO:Specify model dir model_dir = './data/dl_models' model_path = os.path.join( model_dir, '%s_%s.pt' % (self.timestamp, TopKModelSaver.get_configuration_id(self.best_algo_config))) # Remove the old models. if os.path.exists(model_path): os.remove(model_path) mode = 'refit' if self.task_type == IMG_CLS: train_transforms = get_transforms(self.best_algo_config, image_size=self.image_size) dataset.load_data(train_transforms['train'], train_transforms['val']) if dataset.test_data_path is not None: test_transforms = get_test_transforms( self.best_algo_config, image_size=self.image_size) dataset.load_test_data(test_transforms) mode = 'refit_test' else: dataset.load_data() if dataset.test_data_path is not None: dataset.load_test_data() mode = 'refit_test' # Refit the models. _, estimator = get_estimator(self.task_type, config_dict, self.max_epoch, device=self.device) estimator.fit(dataset, mode=mode) # Save to the disk. state = { 'model': estimator.model.state_dict(), 'optimizer': estimator.optimizer_.state_dict(), 'scheduler': estimator.scheduler.state_dict(), 'epoch_num': estimator.epoch_num, 'early_stop': estimator.early_stop } torch.save(state, model_path) else: self.es.refit(dataset)
def predict(self, test_data: DLDataset, mode='test'): predictions = [] cur_idx = 0 num_samples = 0 for algo_id in self.stats["include_algorithms"]: model_configs = self.stats[algo_id]['model_configs'] for idx, config in enumerate(model_configs): if self.task_type == IMG_CLS: test_transforms = get_test_transforms( config, image_size=self.image_size) test_data.load_test_data(test_transforms) test_data.load_data(test_transforms, test_transforms) else: test_data.load_test_data() test_data.load_data() if num_samples == 0: if mode == 'test': dataset = test_data.test_dataset loader = DataLoader(dataset) num_samples = len(loader) else: if test_data.subset_sampler_used: dataset = test_data.train_dataset num_samples = len(test_data.val_sampler) else: dataset = test_data.val_dataset loader = DataLoader(dataset) num_samples = len(loader) estimator = get_estimator_with_parameters(self.task_type, config, self.max_epoch, dataset, self.timestamp, device=self.device) if cur_idx in self.model_idx: if self.task_type in CLS_TASKS: if mode == 'test': predictions.append( estimator.predict_proba( test_data.test_dataset)) else: if test_data.subset_sampler_used: predictions.append( estimator.predict_proba( test_data.train_dataset, sampler=test_data.val_sampler)) else: predictions.append( estimator.predict_proba( test_data.val_dataset)) else: if mode == 'test': predictions.append( estimator.predict(test_data.test_dataset)) else: if test_data.subset_sampler_used: predictions.append( estimator.predict( test_data.train_dataset, sampler=test_data.val_sampler)) else: predictions.append( estimator.predict(test_data.val_dataset)) else: if len(self.shape) == 1: predictions.append(np.zeros(num_samples)) else: predictions.append( np.zeros((num_samples, self.shape[1]))) cur_idx += 1 predictions = np.asarray(predictions) # if predictions.shape[0] == len(self.weights_), # predictions include those of zero-weight models. if predictions.shape[0] == len(self.weights_): return np.average(predictions, axis=0, weights=self.weights_) # if prediction model.shape[0] == len(non_null_weights), # predictions do not include those of zero-weight models. elif predictions.shape[0] == np.count_nonzero(self.weights_): non_null_weights = [w for w in self.weights_ if w > 0] return np.average(predictions, axis=0, weights=non_null_weights) # If none of the above applies, then something must have gone wrong. else: raise ValueError("The dimensions of ensemble predictions" " and ensemble weights do not match!")
def fit(self, train_data: DLDataset, **kwargs): _start_time = time.time() if 'opt_method' in kwargs: self.optalgo = kwargs['opt_method'] else: self.optalgo = 'see' if self.task_type == IMG_CLS: self.image_size = kwargs['image_size'] if self.config_file_path is not None: config_parser = ConfigParser(logger=self.logger) self.update_cs = config_parser.read(self.config_file_path) # TODO: For first-time user, download pretrained params here! algorithm_candidates = self.include_algorithms.copy() num_train_samples = train_data.get_train_samples_num() if self.optalgo == 'hpo': self._fit_in_hpo_way(algorithm_candidates, train_data, **kwargs) return # Initialize solver for each architecture. for estimator_id in self.include_algorithms: cs = self.get_model_config_space(estimator_id) default_config = cs.get_default_configuration() cs.seed(self.seed) hpo_evaluator = DLEvaluator(default_config, self.task_type, max_epoch=self.max_epoch, scorer=self.metric, dataset=train_data, device=self.device, seed=self.seed, timestamp=self.timestamp, **kwargs) optimizer = build_hpo_optimizer(self.evaluation_type, hpo_evaluator, cs, output_dir=self.output_dir, per_run_time_limit=100000, timestamp=self.timestamp, seed=self.seed, n_jobs=self.n_jobs) self.solvers[estimator_id] = optimizer self.evaluators[estimator_id] = hpo_evaluator # Execute profiling procedure. if not self.skip_profile: algorithm_candidates = self.profile_models(num_train_samples) if len(algorithm_candidates) == 0: raise ValueError( 'After profiling, no arch is in the candidates!') else: self.logger.info('After profiling, arch candidates={%s}' % ','.join(algorithm_candidates)) # Execute neural architecture selection. self.logger.info('Before NAS, arch candidates={%s}' % ','.join(algorithm_candidates)) dl_evaluator = DLEvaluator(None, self.task_type, max_epoch=self.max_epoch, scorer=self.metric, dataset=train_data, device=self.device, seed=self.seed, timestamp=self.timestamp, **kwargs) if self.optalgo == 'see': from solnml.components.optimizers.cashp_optimizer import CashpOptimizer self.see_optimizer = CashpOptimizer(self.task_type, algorithm_candidates, self.time_limit, n_jobs=self.n_jobs) inc_config, inc_perf = self.see_optimizer.run(dl_evaluator) self.best_algo_config = inc_config self.best_algo_id = inc_config['estimator'] return algorithm_candidates = self.select_network_architectures( algorithm_candidates, dl_evaluator, num_arch=1, **kwargs) self.logger.info('After NAS, arch candidates={%s}' % ','.join(algorithm_candidates)) # Control flow via round robin. n_algorithm = len(algorithm_candidates) if self.trial_num is None: algo_id = 0 while True: _time_elapsed = time.time() - _start_time if _time_elapsed >= self.time_limit: break _budget_left = self.time_limit - _time_elapsed self.solvers[algorithm_candidates[algo_id]].iterate( budget=_budget_left) algo_id = (algo_id + 1) % n_algorithm else: for id in self.trial_num: self.solvers[algorithm_candidates[id % n_algorithm]].iterate() # Best architecture id. best_scores_ = list() for estimator_id in algorithm_candidates: if estimator_id in self.solvers: solver_ = self.solvers[estimator_id] if len(solver_.perfs) > 0: best_scores_.append(np.max(solver_.perfs)) else: best_scores_.append(-np.inf) else: best_scores_.append(-np.inf) print(algorithm_candidates, best_scores_) assert len(algorithm_candidates) > 0 if len(best_scores_) > 1 and (np.array(best_scores_) > -np.inf).any(): self.best_algo_id = algorithm_candidates[np.argmax(best_scores_)] # Best model configuration. solver_ = self.solvers[self.best_algo_id] inc_idx = np.argmax(solver_.perfs) self.best_algo_config = solver_.configs[inc_idx] else: self.best_algo_id = algorithm_candidates[0] rs = list(self.eval_hist_perfs.keys()) set_flag = False if len(rs) > 0: max_resource = np.max(rs) if max_resource in self.eval_hist_configs: idxs = [ idx for (idx, config) in enumerate( self.eval_hist_configs[max_resource]) if config['estimator'] == self.best_algo_id ] best_idx = np.argmax([ self.eval_hist_perfs[max_resource][idx] for idx in idxs ]) self.best_algo_config = self.eval_hist_configs[ max_resource][best_idx] set_flag = True if not set_flag: solver_ = self.solvers[self.best_algo_id] inc_idx = np.argmax(solver_.perfs) self.best_algo_config = solver_.configs[inc_idx] print(self.best_algo_config) # Skip Ensemble if self.task_type == OBJECT_DET: return if self.ensemble_method is not None: stats = self.fetch_ensemble_members(algorithm_candidates) # Ensembling all intermediate/ultimate models found in above optimization process. self.es = EnsembleBuilder(stats=stats, ensemble_method=self.ensemble_method, ensemble_size=self.ensemble_size, task_type=self.task_type, max_epoch=self.max_epoch, metric=self.metric, timestamp=self.timestamp, device=self.device, output_dir=self.output_dir, **kwargs) self.es.fit(data=train_data)