def _load_default_texts(): """ Loads default general texts Returns ------- result : default 20newsgroup texts """ dataset = Dataset() dataset.fetch_dataset("20NewsGroup") return dataset.get_corpus()
def test_load_20ng(): data_home = get_data_home(data_home=None) cache_path = _pkl_filepath(data_home, "20NewsGroup" + ".pkz") if os.path.exists(cache_path): os.remove(cache_path) dataset = Dataset() dataset.fetch_dataset("20NewsGroup") assert len(dataset.get_corpus()) == 16309 assert len(dataset.get_labels()) == 16309 assert os.path.exists(cache_path) dataset = Dataset() dataset.fetch_dataset("20NewsGroup") assert len(dataset.get_corpus()) == 16309
def test_partitions_fetch(): dataset = Dataset() dataset.fetch_dataset("M10") partitions = dataset.get_partitioned_corpus() assert len(partitions[0]) == 5847 assert len(partitions[1]) == 1254
def test_load_M10(): dataset = Dataset() dataset.fetch_dataset("M10") assert len(set(dataset.get_labels())) == 10
def _restore_parameters(self, name_path): """ Restore the BO parameters from the json file :param name_path: name of the json file :type name_path: str :return: result of BO optimization (scikit-optimize object), surrogate model (scikit-learn object) :rtype: tuple """ # Load the previous results with open(name_path, 'rb') as file: optimization_object = json.load(file) self.search_space = load_search_space(optimization_object["search_space"]) self.acq_func = optimization_object["acq_func"] self.surrogate_model = optimization_object["surrogate_model"] self.kernel = eval(optimization_object["kernel"]) self.optimization_type = optimization_object["optimization_type"] self.model_runs = optimization_object["model_runs"] self.save_models = optimization_object["save_models"] self.save_step = optimization_object["save_step"] self.save_name = optimization_object["save_name"] self.save_models = optimization_object["save_models"] self.save_path = optimization_object["save_path"] self.early_stop = optimization_object["early_stop"] self.early_step = optimization_object["early_step"] self.plot_model = optimization_object["plot_model"] self.plot_best_seen = optimization_object["plot_best_seen"] self.plot_name = optimization_object["plot_name"] self.log_scale_plot = optimization_object["log_scale_plot"] self.random_state = optimization_object["random_state"] self.dict_model_runs = optimization_object['dict_model_runs'] self.number_of_previous_calls = optimization_object['current_call'] + 1 self.current_call = optimization_object['current_call'] + 1 self.number_of_call = optimization_object['number_of_call'] self.save_path = optimization_object['save_path'] self.x0 = optimization_object['x0'] self.y0 = optimization_object['y0'] self.n_random_starts = optimization_object['n_random_starts'] self.initial_point_generator = optimization_object['initial_point_generator'] self.topk = optimization_object['topk'] self.time_eval = optimization_object["time_eval"] res = None # Load the dataset dataset = Dataset() if not optimization_object["is_cached"]: dataset.load_custom_dataset_from_folder(optimization_object["dataset_path"]) else: dp = optimization_object["dataset_path"][:-(len(optimization_object["dataset_name"]) + len("_py3.pkz"))] dataset.fetch_dataset(optimization_object["dataset_name"], data_home=dp) self.dataset = dataset # Load the metric self._load_metric(optimization_object, dataset) # Load the model self.model = load_model(optimization_object) # Creation of the hyperparameters self.hyperparameters = list(sorted(self.search_space.keys())) # Choice of the optimizer opt = choose_optimizer(self) # Update number_of_call for restarting for i in range(self.number_of_previous_calls): next_x = [optimization_object["x_iters"][key][i] for key in self.hyperparameters] f_val = -optimization_object["f_val"][i] if self.optimization_type == 'Maximize' else \ optimization_object["f_val"][i] res = opt.tell(next_x, f_val) # Create the directory where the results are saved Path(self.save_path).mkdir(parents=True, exist_ok=True) self.model_path_models = self.save_path + "models/" return res, opt