def __init__(self, **kwargs):
self.classifier_type = kwargs.get('classifier_type') or 'GBC'
if self.classifier_type not in CLASSIFIER_TYPE_TO_PARAMS.keys():
raise Exception("classifier_type must be one of %s" % CLASSIFIER_TYPE_TO_PARAMS.keys())
self.client_token = kwargs.get('client_token')
self.dataset_name = kwargs.get('dataset_name')
self.test_set_size = kwargs.get('test_set_size')
self.num_sigopt_suggestions = kwargs.get('num_sigopt_suggestions') or NUM_SIGOPT_SUGGESTIONS
self.grid_search_width = kwargs.get('grid_search_width') or GRID_SEARCH_WIDTH
self.num_random_searches = kwargs.get('num_random_searches') or NUM_RANDOM_SEARCHES
self.dataset = self._load_dataset()
def __init__(self, **kwargs):
self.classifier_type = kwargs.get('classifier_type') or 'GBC'
if self.classifier_type not in CLASSIFIER_TYPE_TO_PARAMS.keys():
raise Exception("classifier_type must be one of %s" % CLASSIFIER_TYPE_TO_PARAMS.keys())
self.client_token = client_token
self.dataset_name = kwargs.get('dataset_name')
self.test_set_size = kwargs.get('test_set_size')
self.num_sigopt_suggestions = kwargs.get('num_sigopt_suggestions') or NUM_SIGOPT_SUGGESTIONS
self.grid_search_width = kwargs.get('grid_search_width') or GRID_SEARCH_WIDTH
self.num_random_searches = kwargs.get('num_random_searches') or NUM_RANDOM_SEARCHES
self.dataset = self._load_dataset()
fout,
sigopt_post=sigopt_post)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Classifier Tuner')
parser.add_argument(
'--client-token',
type=str,
help='Your sigopt API token. Get this from https://sigopt.com/tokens',
required=True,
)
parser.add_argument(
'--classifier-type',
type=str,
choices=CLASSIFIER_TYPE_TO_PARAMS.keys(),
help='The type of classifier to use. Defaults to GBC.',
default='GBC',
)
parser.add_argument(
'--dataset-name',
type=str,
help='The sklearn dataset to use. Defaults to datasets.load_digits().',
)
parser.add_argument(
'--test-set-size',
type=int,
help=
'The number of points in the test set. The remainder of the dataset will be the test set.',
)
parser.add_argument(
def calculate_objective(self, assignments):
"""Return the fit of the classifier with the given hyperparameters and the test data."""
classifier = self.get_classifier(assignments)
classifier.fit(self.dataset.X_train, self.dataset.y_train)
return classifier.score(self.dataset.X_test, self.dataset.y_test)
def run_example(self, experiment, generator, sigopt_post=False, output_file=None):
"""Test various hyperparameter configurations against the dataset given a generator."""
with open(output_file, 'w') as fout:
for assignments in generator(experiment):
score = self.calculate_objective(assignments)
self.output_score(experiment, assignments, score, fout, sigopt_post=sigopt_post)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Classifier Tuner')
parser.add_argument('--classifier-type', type=str, choices=CLASSIFIER_TYPE_TO_PARAMS.keys(), help='The type of classifier to use. Defaults to GBC')
parser.add_argument('--dataset-name', type=str, help='The sklearn dataset to use. Defaults to datasets.load_digits().')
parser.add_argument('--test-set-size', type=int, help='The number of points in the test set. The remainder of the dataset will be the test set.')
parser.add_argument('--num-sigopt-suggestions', type=int, help='The number of suggestions to request from SigOpt.')
parser.add_argument('--grid-search-width', type=int, help='How many grid points in each dimension to use for grid search')
parser.add_argument('--num-random-searches', type=int, help='How many random search parameter configurations to test')
args = vars(parser.parse_args())
try:
runner = ExampleRunner(**args)
experiment = runner.create_experiment()
print('Running SigOpt...')
runner.run_example(
experiment,
runner.sigopt_generator,
return classifier.score(self.dataset.X_test, self.dataset.y_test)
def run_example(self, experiment, generator, sigopt_post=False, output_file=None):
"""Test various hyperparameter configurations against the dataset given a generator."""
with open(output_file, "w") as fout:
for assignments in generator(experiment):
score = self.calculate_objective(assignments)
self.output_score(experiment, assignments, score, fout, sigopt_post=sigopt_post)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Classifier Tuner")
parser.add_argument(
"--classifier-type",
type=str,
choices=CLASSIFIER_TYPE_TO_PARAMS.keys(),
help="The type of classifier to use. Defaults to GBC",
)
parser.add_argument(
"--dataset-name", type=str, help="The sklearn dataset to use. Defaults to datasets.load_digits()."
)
parser.add_argument(
"--test-set-size",
type=int,
help="The number of points in the test set. The remainder of the dataset will be the test set.",
)
parser.add_argument("--num-sigopt-suggestions", type=int, help="The number of suggestions to request from SigOpt.")
parser.add_argument(
"--grid-search-width", type=int, help="How many grid points in each dimension to use for grid search"
)
parser.add_argument(
