def __init__(self, **kwargs): self.classifier_type = kwargs.get('classifier_type') or 'GBC' if self.classifier_type not in CLASSIFIER_TYPE_TO_PARAMS.keys(): raise Exception("classifier_type must be one of %s" % CLASSIFIER_TYPE_TO_PARAMS.keys()) self.client_token = kwargs.get('client_token') self.dataset_name = kwargs.get('dataset_name') self.test_set_size = kwargs.get('test_set_size') self.num_sigopt_suggestions = kwargs.get('num_sigopt_suggestions') or NUM_SIGOPT_SUGGESTIONS self.grid_search_width = kwargs.get('grid_search_width') or GRID_SEARCH_WIDTH self.num_random_searches = kwargs.get('num_random_searches') or NUM_RANDOM_SEARCHES self.dataset = self._load_dataset()
def __init__(self, **kwargs): self.classifier_type = kwargs.get('classifier_type') or 'GBC' if self.classifier_type not in CLASSIFIER_TYPE_TO_PARAMS.keys(): raise Exception("classifier_type must be one of %s" % CLASSIFIER_TYPE_TO_PARAMS.keys()) self.client_token = client_token self.dataset_name = kwargs.get('dataset_name') self.test_set_size = kwargs.get('test_set_size') self.num_sigopt_suggestions = kwargs.get('num_sigopt_suggestions') or NUM_SIGOPT_SUGGESTIONS self.grid_search_width = kwargs.get('grid_search_width') or GRID_SEARCH_WIDTH self.num_random_searches = kwargs.get('num_random_searches') or NUM_RANDOM_SEARCHES self.dataset = self._load_dataset()
fout, sigopt_post=sigopt_post) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Classifier Tuner') parser.add_argument( '--client-token', type=str, help='Your sigopt API token. Get this from https://sigopt.com/tokens', required=True, ) parser.add_argument( '--classifier-type', type=str, choices=CLASSIFIER_TYPE_TO_PARAMS.keys(), help='The type of classifier to use. Defaults to GBC.', default='GBC', ) parser.add_argument( '--dataset-name', type=str, help='The sklearn dataset to use. Defaults to datasets.load_digits().', ) parser.add_argument( '--test-set-size', type=int, help= 'The number of points in the test set. The remainder of the dataset will be the test set.', ) parser.add_argument(
def calculate_objective(self, assignments): """Return the fit of the classifier with the given hyperparameters and the test data.""" classifier = self.get_classifier(assignments) classifier.fit(self.dataset.X_train, self.dataset.y_train) return classifier.score(self.dataset.X_test, self.dataset.y_test) def run_example(self, experiment, generator, sigopt_post=False, output_file=None): """Test various hyperparameter configurations against the dataset given a generator.""" with open(output_file, 'w') as fout: for assignments in generator(experiment): score = self.calculate_objective(assignments) self.output_score(experiment, assignments, score, fout, sigopt_post=sigopt_post) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Classifier Tuner') parser.add_argument('--classifier-type', type=str, choices=CLASSIFIER_TYPE_TO_PARAMS.keys(), help='The type of classifier to use. Defaults to GBC') parser.add_argument('--dataset-name', type=str, help='The sklearn dataset to use. Defaults to datasets.load_digits().') parser.add_argument('--test-set-size', type=int, help='The number of points in the test set. The remainder of the dataset will be the test set.') parser.add_argument('--num-sigopt-suggestions', type=int, help='The number of suggestions to request from SigOpt.') parser.add_argument('--grid-search-width', type=int, help='How many grid points in each dimension to use for grid search') parser.add_argument('--num-random-searches', type=int, help='How many random search parameter configurations to test') args = vars(parser.parse_args()) try: runner = ExampleRunner(**args) experiment = runner.create_experiment() print('Running SigOpt...') runner.run_example( experiment, runner.sigopt_generator,
return classifier.score(self.dataset.X_test, self.dataset.y_test) def run_example(self, experiment, generator, sigopt_post=False, output_file=None): """Test various hyperparameter configurations against the dataset given a generator.""" with open(output_file, "w") as fout: for assignments in generator(experiment): score = self.calculate_objective(assignments) self.output_score(experiment, assignments, score, fout, sigopt_post=sigopt_post) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Classifier Tuner") parser.add_argument( "--classifier-type", type=str, choices=CLASSIFIER_TYPE_TO_PARAMS.keys(), help="The type of classifier to use. Defaults to GBC", ) parser.add_argument( "--dataset-name", type=str, help="The sklearn dataset to use. Defaults to datasets.load_digits()." ) parser.add_argument( "--test-set-size", type=int, help="The number of points in the test set. The remainder of the dataset will be the test set.", ) parser.add_argument("--num-sigopt-suggestions", type=int, help="The number of suggestions to request from SigOpt.") parser.add_argument( "--grid-search-width", type=int, help="How many grid points in each dimension to use for grid search" ) parser.add_argument(