def test_create(self):
conn = Connection(client_token='client_token')
assert conn.impl.api_url == 'https://api.sigopt.com'
assert conn.impl.requestor.verify_ssl_certs is True
assert conn.impl.requestor.proxies is None
assert isinstance(conn.clients, ApiResource)
assert isinstance(conn.experiments, ApiResource)
def test_create(self): conn = Connection(client_token='client_token', _show_deprecation_warning=False) assert conn.impl.api_url == 'https://api.sigopt.com' assert conn.impl.requestor.verify_ssl_certs is None assert conn.impl.requestor.proxies is None assert conn.impl.requestor.timeout == DEFAULT_HTTP_TIMEOUT assert conn.impl.requestor.auth.username == 'client_token' assert isinstance(conn.clients, ApiResource) assert isinstance(conn.experiments, ApiResource)
def test_create_uses_session_if_provided(self):
session = mock.Mock()
conn = Connection(client_token='client_token', session=session, _show_deprecation_warning=False)
assert conn.impl.requestor.session is session
response = mock.Mock()
session.request.return_value = response
response.status_code = 200
response.text = '{}'
session.request.assert_not_called()
conn.experiments().fetch()
session.request.assert_called_once()
def __init__(self,
api_token=None,
parameter_domain=None,
maze_size=(30, 30),
num_tests=100,
down_prob=1.0):
"""Create a new SigOpt testing tool.
The api_token is a long string which can be found on your account at www.sigopt.com/tokens.
You can either pass that string here or you can modify this file to save it on your local machine.
parameter_domain describes the allowable values for the relative probabilities of constructing the maze with
"left", "up" or "right" moves. These probabilities are relative to the probability of moving "down" which
is always fixed at 1.0 (by default). The domain should be a numpy.array of shape (3, 2), where:
* the first row denotes the lower and upper bound for the "left" domain,
* the second row denotes the lower and upper bound for the "up" domain,
* the third row denotes the lower and upper bound for the "right" domain,
:param api_token: The API token used to access SigOpt (or SIGOPT_API_TOKEN can be modified above)
:type api_token: str
:param parameter_domain: Domain on which the solver should consider the three parameters
:type parameter_domain: numpy.array of shape (3, 2)
:param maze_size: size of the maze to randomly generate and test (must be at least 2 rows and 2 columns)
:type maze_size: tuple of length 2
:param num_tests: How many random mazes should be constructed to estimate the mean and standard deviation
:type num_tests: int > 1, (will default to the value stored at construction)
:param down_prob: The relative probability of moving down (against which other probabilities are compared)
:type down_prob: float > 0
"""
self.conn = Connection(client_token=api_token or SIGOPT_API_TOKEN)
self.experiment = None
self.domain = parameter_domain or numpy.array([[.01, 100]] * 3)
if not numpy.all(self.domain[:, 1] > self.domain[:, 0]):
raise AssertionError(
'The minimum values (column 1) in the domain must be less than the maximum (column 2)'
)
assert type(maze_size) in (list,
tuple) and len(maze_size) == 2 and all(
[m > 1 for m in maze_size])
self.maze_size = maze_size
assert type(num_tests) == int and num_tests > 1
self.num_tests = num_tests
assert down_prob > 0
self.down_prob = float(down_prob)
def create_experiment(self):
"""Create a SigOpt experiment for optimizing the classifier hyperparameters."""
conn = Connection(client_token=self.client_token)
params = CLASSIFIER_TYPE_TO_PARAMS[self.classifier_type]
try:
return conn.experiments().create(
name="Example Classifier",
parameters=params,
)
except ApiException as e:
if e.status_code == 403 and '*****@*****.**' in str(e):
existing_experiments = conn.experiments().fetch().data
if existing_experiments:
raise Exception(
"You have existing experiments on sigopt.com: {0}."
" You have exceeded the number of experiments that can be created under your plan."
" Please visit https://sigopt.com/pricing to learn about plans."
.format([
'https://sigopt.com/experiment/{0}'.format(e.id)
for e in existing_experiments
]))
raise
def output_score(self,
experiment,
assignments,
score,
fout,
sigopt_post=False):
"""Log the score, optionally save it to file, and/or report it back to SigOpt."""
suggestion = [
assignments[param.name] for param in experiment.parameters
]
output = "score: {suggestion} = {score}\n".format(
suggestion=tuple(suggestion), score=score)
print(output, end='')
fout.write(output)
if sigopt_post is True:
conn = Connection(client_token=self.client_token)
conn.experiments(experiment.id).observations().create(
assignments=assignments,
value=score,
)
conn.experiments(experiment.id).suggestions().delete()
def sigopt_generator(self, experiment):
"""Generate optimal parameter configurations using SigOpt."""
for _ in xrange(NUM_SIGOPT_SUGGESTIONS):
conn = Connection(client_token=self.client_token)
suggestion = conn.experiments(experiment.id).suggestions().create()
yield suggestion.assignments.to_json()
def test_verify(self):
conn = Connection('client_token', _show_deprecation_warning=False)
conn.set_verify_ssl_certs(False)
assert conn.impl.requestor.verify_ssl_certs is False
def test_timeout(self):
conn = Connection('client_token')
conn.set_timeout(30)
assert conn.impl.requestor.timeout == 30
(stdoutdata,stderrdata) = process.communicate()
sys.stderr.write(stderrdata)
return float(stdoutdata.strip())
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--command', required=True, help="The command to run the function whose parameters you would "
"like to optimize. Should accept parameters as command line argument and output only the evaluated metric at the "
"suggested point.")
parser.add_argument('--experiment_id', required=True, help="The parameters of this experiment should be the "
"same type and name of the command line arguments to your executable file.")
parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/user/profile")
the_args = parser.parse_args()
connection = Connection(client_token=the_args.client_token)
experiment = connection.experiments(the_args.experiment_id).fetch()
connection.experiments(the_args.experiment_id).suggestions().delete(state="open")
evaluator = SubProcessEvaluator(the_args.command)
# In a loop: receive a suggestion, evaluate the metric, report an observation
while True:
suggestion = connection.experiments(experiment.id).suggestions().create()
print('Evaluating at suggested assignments: {0}'.format(suggestion.assignments))
value = evaluator.evaluate_metric(suggestion.assignments)
print('Reporting observation of value: {0}'.format(value))
connection.experiments(experiment.id).observations().create(
suggestion=suggestion.id,
value=value,
)
def test_api_url_env(self):
with mock.patch.dict(os.environ,
{'SIGOPT_API_URL': 'https://api-env.sigopt.com'}):
conn = Connection('client_token')
assert conn.impl.api_url == 'https://api-env.sigopt.com'
def test_error(self):
with mock.patch.dict(os.environ, {'SIGOPT_API_TOKEN': ''}):
with pytest.raises(ValueError):
Connection()
def test_error(self):
with pytest.raises(ValueError):
Connection()
def test_create(self):
conn = Connection(client_token='client_token')
conn.set_api_url('https://api-test.sigopt.com')
assert isinstance(conn.clients, ApiResource)
assert isinstance(conn.experiments, ApiResource)
def test_verify(self):
conn = Connection('client_token')
conn.set_verify_ssl_certs(False)
assert conn.impl.requestor.verify_ssl_certs is False
def test_environment_variable(self):
with mock.patch.dict(os.environ, {'SIGOPT_API_TOKEN': 'client_token'}):
Connection()
def test_api_url(self):
conn = Connection('client_token', _show_deprecation_warning=False)
conn.set_api_url('https://api-test.sigopt.com')
assert conn.impl.api_url == 'https://api-test.sigopt.com'
def test_token_in_config(self, config_dict):
with mock.patch.dict(config_dict, {'api_token': 'test_token_in_config'}), mock.patch.dict(os.environ, {}):
conn = Connection(_show_deprecation_warning=False)
assert conn.impl.requestor.auth.username == 'test_token_in_config'
def test_environment_variable(self):
with mock.patch.dict(os.environ, {'SIGOPT_API_TOKEN': 'client_token'}):
conn = Connection(_show_deprecation_warning=False)
assert conn.impl.requestor.auth.username == 'client_token'
def test_api_url(self):
conn = Connection('client_token')
conn.set_api_url('https://api-test.sigopt.com')
assert conn.impl.api_url == 'https://api-test.sigopt.com'
def test_proxies(self):
conn = Connection('client_token', _show_deprecation_warning=False)
conn.set_proxies({'http': 'http://127.0.0.1:6543'})
assert conn.impl.requestor.proxies['http'] == 'http://127.0.0.1:6543'
def test_proxies(self):
conn = Connection('client_token')
conn.set_proxies({'http': 'http://127.0.0.1:6543'})
assert conn.impl.requestor.proxies['http'] == 'http://127.0.0.1:6543'
def test_error(self):
with mock.patch.dict(os.environ, {'SIGOPT_API_TOKEN': ''}):
with pytest.raises(ValueError):
Connection(_show_deprecation_warning=False)
def test_timeout(self):
conn = Connection('client_token', _show_deprecation_warning=False)
conn.set_timeout(30)
assert conn.impl.requestor.timeout == 30
def test_environment_variable(self): os.environ['SIGOPT_API_TOKEN'] = 'client_token' Connection() del os.environ['SIGOPT_API_TOKEN']
