class ExampleRunner(threading.Thread):
def __init__(self, client_token, experiment_id):
threading.Thread.__init__(self)
self.connection = Connection(client_token=client_token)
self.experiment_id = experiment_id
def run(self):
while True:
suggestion = self.connection.experiments(self.experiment_id).suggestions().create()
print('{0} - Evaluating at parameters: {1}'.format(threading.current_thread(), suggestion.assignments))
value = self.evaluate_metric(suggestion.assignments)
print('{0} - Observed value: {1}'.format(threading.current_thread(), value))
self.connection.experiments(self.experiment_id).observations().create(
suggestion=suggestion.id,
value=value,
)
def evaluate_metric(self, assignments):
"""
Replace this with the function you want to optimize
This fictitious example has only two parameters, named x1 and x2
"""
sleep_seconds = 10
print('{0} - Sleeping for {1} seconds to simulate expensive computation...'.format(threading.current_thread(), sleep_seconds))
time.sleep(sleep_seconds)
return sigopt.examples.franke_function(assignments['x1'], assignments['x2'])
def __init__(self,
evalfun,
exptDetail,
name,
gpus,
exptId=None,
resume=False,
**kwargs):
threading.Thread.__init__(
self) # required: call constructor of threading class
self.evalfun = evalfun
self.gpus = gpus
self.conn = Connection(
client_token=api_key) # start connection to sigopt
if exptId == None: # start anew experiment
expt = self.conn.experiments().create(**exptDetail)
exptId = expt.id
else: # use existing experiment id
expt = self.conn.experiments(exptId)
expt.update(**exptDetail)
if not resume: # delete all observations and suggestions
expt.observations().delete()
expt.suggestions().delete()
self.name = name
self.exptId = exptId # save expt id in the instance
print(
"View your experiment progress: https://sigopt.com/experiment/{}".
format(self.exptId))
def run_sigopt_optimization():
conn = Connection(client_token=os.environ['SIGOPT_TOKEN'])
# experiment = conn.experiments().create(
# name='Simple CNN Sentiment Analysis Hyperparameter Search',
# parameters=[
# dict(name='top_words', type='int', bounds=dict(min=100, max=88000)),
# dict(name='max_words', type='int', bounds=dict(min=10, max=1000)),
# dict(name='filters', type='int', bounds=dict(min=0, max=10)),
# dict(name='kernel_size', type='int', bounds=dict(min=1, max=5)),
# dict(name='pool_size', type='int', bounds=dict(min=2, max=8)),
# dict(name='dense_size', type='int', bounds=dict(min=2, max=1024)),
# dict(name='batch_size', type='int', bounds=dict(min=2, max=1024)),
# ],
# )
# print("Created experiment: https://sigopt.com/experiment/" + experiment.id)
e_id = str(48737)
for _ in range(300):
suggestion = conn.experiments(e_id).suggestions().create()
options = suggestion.assignments
s = SentimentAnalyzer(options)
s.train_classifier()
value = s.evaluate_classifier()
conn.experiments(e_id).observations().create(
suggestion=suggestion.id,
value=value[1],
)
def __init__(self, params):
super().__init__(params)
self.experiment_id = os.getenv("SIGOPT_EXPERIMENT", None)
api_token = os.getenv("SIGOPT_API_TOKEN")
if api_token is None:
raise ValueError(
"envvar SIGOPT_API_TOKEN must be set with SigOptSolver")
self.conn = Connection(client_token=api_token)
self.logger = logging.getLogger(
".".join(__name__.split(".")[:-1] + ["SigOptSolver"]))
if self.experiment_id is None:
params = flatten_params(params)
experiment = self.conn.experiments().create(
name=f'Mincall opt {socket.gethostname()}',
parameters=[{
"name": name,
"type": p.type,
"bounds": {
"min": p.min,
"max": p.max
}
} for name, p in params.items()],
)
self.experiment_id = experiment.id
self.logger.info(
f"Created experiment: https://sigopt.com/experiment/{self.experiment_id} in "
f"{'DEVELOPMENT' if experiment.development else 'PRODUCTION'} mode"
)
else:
self.logger.info(
"Using experiment: https://sigopt.com/experiment/" +
self.experiment_id)
def __init__(self, evalfun, exptId, name, gpu):
threading.Thread.__init__(self)
self.conn = Connection(client_token=api_key)
self.exptId = exptId
self.name = name
self.gpu = gpu
self.evalfun = evalfun
class ExampleRunner(threading.Thread):
def __init__(self, client_token, experiment_id):
threading.Thread.__init__(self)
self.connection = Connection(client_token=client_token)
self.experiment_id = experiment_id
def run(self):
while True:
suggestion = self.connection.experiments(self.experiment_id).suggestions().create()
print('{0} - Evaluating at parameters: {1}'.format(threading.current_thread(), suggestion.assignments))
value = self.evaluate_metric(suggestion.assignments)
print('{0} - Observed value: {1}'.format(threading.current_thread(), value))
self.connection.experiments(self.experiment_id).observations().create(
suggestion=suggestion.id,
value=value,
)
def evaluate_metric(self, assignments):
"""
Replace this with the function you want to optimize
This fictitious example has only two parameters, named x1 and x2
"""
sleep_seconds = 10
print('{0} - Sleeping for {1} seconds to simulate expensive computation...'.format(threading.current_thread(), sleep_seconds))
time.sleep(sleep_seconds)
return sigopt.examples.franke_function(assignments['x1'], assignments['x2'])
def create_experiment(apikey):
conn = Connection(client_token=apikey)
experiment = conn.experiments().create(
name="Coronaboard -- AutoARIMA",
parameters=[
dict(name="max_p", bounds=dict(min=14, max=30), type="int"),
dict(name="max_q", bounds=dict(min=14, max=30), type="int"),
dict(name="max_d", bounds=dict(min=14, max=30), type="int"),
dict(name="maxiter", bounds=dict(min=80, max=360), type="int"),
dict(name="scoring",
categorical_values=[dict(name="mse"),
dict(name="mae")],
type="categorical"),
dict(name="information_criterion",
categorical_values=[
dict(name="aic"),
dict(name="bic"),
dict(name="hqic"),
dict(name="oob")
],
type="categorical")
],
metrics=[dict(name="SMAPE", objective="minimize")],
metadata=dict(template="dashboard"),
observation_budget=400,
parallel_bandwidth=12,
project="dashboard-dev")
return experiment.id
def create_experiment_lda(apikey):
conn = Connection(client_token=apikey)
experiment = conn.experiments().create(
name="Coronavirus Antivirals -- LDA",
parameters=[
dict(name="topics", bounds=dict(min=16, max=512), type="int"),
dict(name="estimators", bounds=dict(min=256, max=512), type="int"),
dict(name="topic_epochs", bounds=dict(min=1, max=6), type="int"),
dict(name="decay", bounds=dict(min=0.5, max=0.999), type="double"),
dict(name="max_vocab",
bounds=dict(min=10000, max=100000),
type="int"),
dict(name="min_samples_split",
bounds=dict(min=4, max=18),
type="int"),
dict(name="min_samples_leaf",
bounds=dict(min=2, max=4),
type="int"),
dict(name="topic_iterations",
bounds=dict(min=20, max=200),
type="int"),
dict(name="tree_criterion",
categorical_values=[dict(name="gini"),
dict(name="entropy")],
type="categorical")
],
metrics=[
dict(name="AUC", objective="maximize"),
dict(name="Model ID", strategy="store")
],
metadata=dict(template="antivirals"),
observation_budget=300,
parallel_bandwidth=10,
project="antivirals")
return experiment.id
def __init__(self, experiment_id): """ Initialize a worker thread, creating the SigOpt API connection and storing the previously created experiment's id """ threading.Thread.__init__(self) self.experiment_id = experiment_id self.conn = Connection(client_token=SIGOPT_API_KEY)
class Master(threading.Thread):
'''master thread which initializes an experimetn and starts the workers (passing in the expt id and gpu it should run on.
doesnt seem necessary for this to be its own class nor does it seem necessary for it to inhereit the threading class.'''
def __init__(self,
evalfun,
exptDetail,
name,
gpus,
exptId=None,
resume=False,
**kwargs):
threading.Thread.__init__(
self) # required: call constructor of threading class
self.evalfun = evalfun
self.gpus = gpus
self.conn = Connection(
client_token=api_key) # start connection to sigopt
if exptId == None: # start anew experiment
expt = self.conn.experiments().create(**exptDetail)
exptId = expt.id
else: # use existing experiment id
expt = self.conn.experiments(exptId)
expt.update(**exptDetail)
if not resume: # delete all observations and suggestions
expt.observations().delete()
expt.suggestions().delete()
self.name = name
self.exptId = exptId # save expt id in the instance
print(
"View your experiment progress: https://sigopt.com/experiment/{}".
format(self.exptId))
@property
def remaining_observations(self):
experiment = self.conn.experiments(self.exptId).fetch()
return experiment.observation_budget - experiment.progress.observation_count
def run(self):
'''start the workers and evaluations'''
tries = 3
while (tries > 0 and self.remaining_observations > 0):
workers = [
Worker(self.evalfun,
exptId=self.exptId,
name=self.name,
gpu=gpu) for gpu in self.gpus
]
for worker in workers:
worker.start(
) # each worker will communicated spearately with sigopt and will not communicate wiht one another
for worker in workers:
worker.join(
) # wait till all workers are done, which only happens when observation budget runs out
self.conn.experiments(
self.exptId).suggestions().delete(state='open')
tries -= 1 # will reach this point only when observation budget runs out or all workers have failed
def continue_experiment(apikey, exp_id):
counties_url = 'https://github.com/nytimes/covid-19-data/raw/42378bc95a04ff4fe798d2378affb351954db164/us-counties.csv'
us_counties = get_counties_data(counties_url)
conn = Connection(client_token=apikey)
experiment = conn.experiments(exp_id).fetch()
for _ in range(experiment.observation_budget):
try:
suggestion = conn.experiments(exp_id).suggestions().create()
except ApiException:
conn.experiments(exp_id).suggestions().delete()
suggestion = conn.experiments(exp_id).suggestions().create()
assignments = suggestion.assignments
print(f"Hyperpameters: {assignments}")
try:
_, _, metrics = forecast(us_counties,
log_metrics=True,
hp={
**HYPERPARAMETERS,
**assignments
})
mean = sum(metrics.values()) / len(metrics)
except Exception as e:
print("Invalid hyperparameter combination!")
print(str(e))
mean = 0.9999
finally:
conn.experiments(exp_id).observations().create(
suggestion=suggestion.id, value=mean)
def continue_experiment(dbstring, apikey, exp_id):
conn = Connection(client_token=apikey)
experiment = conn.experiments(exp_id).fetch()
for _ in range(experiment.observation_budget):
try:
suggestion = conn.experiments(exp_id).suggestions().create()
except ApiException:
conn.experiments(exp_id).suggestions().delete()
suggestion = conn.experiments(exp_id).suggestions().create()
if suggestion.assignments.get('topics'):
suggestion.assignments['vector_algo'] = 'lda'
elif suggestion.assignments.get('vec_dims'):
suggestion.assignments['vector_algo'] = 'doc2vec'
assignments = Hyperparameters.from_dict(suggestion.assignments)
chem = run_train_models(dbstring, assignments)
mean = sum(chem.toxicity.auc) / len(chem.toxicity.auc)
conn.experiments(exp_id).observations().create(
suggestion=suggestion.id,
values=[
dict(
name="AUC",
value=mean,
),
dict(name="Model ID", value=UUID(chem.uuid).int)
])
class Worker(threading.Thread):
"""
Shows what a worker machine does when running SigOpt in a distributed setting.
"""
def __init__(self, experiment_id):
"""
Initialize a worker thread, creating the SigOpt API connection and storing the previously
created experiment's id
"""
threading.Thread.__init__(self)
self.experiment_id = experiment_id
self.conn = Connection(client_token=SIGOPT_API_KEY)
@property
def metadata(self):
"""
Use metadata to keep track of the host that each Suggestion and Observation is created on.
Learn more: https://sigopt.com/docs/overview/metadata
"""
return dict(host=threading.current_thread().name)
@property
def remaining_observations(self):
"""
Re-fetch the experiment and calculate how many Observations we need to run until
we reach the observation budget
"""
experiment = self.conn.experiments(self.experiment_id).fetch()
return experiment.observation_budget - experiment.progress.observation_count
def run(self):
"""
SigOpt acts as the scheduler for the Suggestions, so all you need to do is run the
optimization loop until there are no remaining Observations to be reported.
We handle exceptions by reporting failed Observations. Learn more about handling
failure cases: https://sigopt.com/docs/overview/metric_failure
"""
while self.remaining_observations > 0:
suggestion = self.conn.experiments(
self.experiment_id).suggestions().create(
metadata=self.metadata)
try:
value = evaluate_model(suggestion.assignments)
failed = False
except Exception:
value = None
failed = True
self.conn.experiments(self.experiment_id).observations().create(
suggestion=suggestion.id,
value=value,
failed=failed,
metadata=self.metadata,
)
def __init__(self):
"""Constructor."""
self.logger = logging.getLogger(__name__) # get the logger!
self.conn = Connection(
client_token="XWCROUDALHMNJFABTLYVXBUHISZQKKACUGULCENHPSZNQPSD")
self.conn.set_api_url("https://api.sigopt.com")
self.experiment = None
self.suggestion = None
self.model = None
self.acc = None
def __init__(self, experiment_id=None):
self.conn = Connection(client_token=SIG_KEY)
if experiment_id is None:
experiment = self.conn.experiments().create(
name='new_york_city',
parameters=PARAMS,
metrics=METRICS,
observation_budget=BUDGET,
project='pandemic')
self.experiment_id = experiment.id
else:
self.experiment_id = experiment_id
class SigOpt(AbstractSolver):
def __init__(self, params):
super().__init__(params)
self.experiment_id = os.getenv("SIGOPT_EXPERIMENT", None)
api_token = os.getenv("SIGOPT_API_TOKEN")
if api_token is None:
raise ValueError(
"envvar SIGOPT_API_TOKEN must be set with SigOptSolver")
self.conn = Connection(client_token=api_token)
self.logger = logging.getLogger(
".".join(__name__.split(".")[:-1] + ["SigOptSolver"]))
if self.experiment_id is None:
params = flatten_params(params)
experiment = self.conn.experiments().create(
name=f'Mincall opt {socket.gethostname()}',
parameters=[{
"name": name,
"type": p.type,
"bounds": {
"min": p.min,
"max": p.max
}
} for name, p in params.items()],
)
self.experiment_id = experiment.id
self.logger.info(
f"Created experiment: https://sigopt.com/experiment/{self.experiment_id} in "
f"{'DEVELOPMENT' if experiment.development else 'PRODUCTION'} mode"
)
else:
self.logger.info(
"Using experiment: https://sigopt.com/experiment/" +
self.experiment_id)
def new_assignment(self):
suggestion = self.conn.experiments(
self.experiment_id).suggestions().create()
return Assignment(
params=unflatten_params(suggestion.assignments),
name=f"{suggestion.id}-{name_generator()}",
context=suggestion.id,
)
def report(self, assignment: Assignment, observation: Observation):
self.logger.info(
f"Assignment:\n{pformat(dict(assignment._asdict()))}\n"
f"has observation:\n{pformat(dict(observation._asdict()))}")
self.conn.experiments(self.experiment_id).observations().create(
suggestion=assignment.context,
value=observation.metric,
value_stddev=observation.metric_std,
metadata=toolz.merge(observation.metadata,
{'model_name': assignment.name}))
def sigopt_cube(objective, scale, n_trials):
"""
:param objective: returns 1-tuple
:param scale: float defines cubical domain
:param n_trials: int
:return:
"""
params = [{'name': 'u1', 'type': 'double', 'bounds': {'min': -scale, 'max': scale}},
{'name': 'u2', 'type': 'double', 'bounds': {'min': -scale, 'max': scale}},
{'name': 'u3', 'type': 'double', 'bounds': {'min': -scale, 'max': scale}}]
metrics = [{'name': 'slimy_moose', 'objective': 'minimize'}]
conn = Connection(client_token=SIG_KEY)
experiment = conn.experiments().create(name='exp_'+str(uuid.uuid4())[:6],
parameters=params,
metrics=metrics,
observation_budget=n_trials,
project=PROJECT)
def _objective(assignments):
u = [assignments['u1'], assignments['u2'], assignments['u3']]
return objective(u)[0]
while experiment.progress.observation_count < experiment.observation_budget:
suggestion = conn.experiments(experiment.id).suggestions().create()
value = _objective(assignments=suggestion.assignments)
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id,
value=value,
)
experiment = conn.experiments(experiment.id).fetch()
all_best_assignments = conn.experiments(experiment.id).best_assignments().fetch()
return all_best_assignments.data[0].value
class Master(threading.Thread):
"""
Shows what a master machine does when running SigOpt in a distributed setting.
"""
def __init__(self):
"""
Initialize the master thread, creating the SigOpt API connection and the experiment.
We use the observation_budget field on the experiment to keep track of approximately
how many total Observations we want to report. We recommend using a number between 10-20x
the number of parameters in an experiment.
"""
threading.Thread.__init__(self)
self.conn = Connection(client_token=SIGOPT_API_KEY)
experiment = self.conn.experiments().create(
name='Parallel Experiment',
project='sigopt-examples',
parameters=PARAMETERS,
metrics=[dict(name='cv_accuracies', objective='maximize')],
observation_budget=len(PARAMETERS) * 20,
)
print(
"View your experiment progress: https://sigopt.com/experiment/{}".
format(experiment.id))
self.experiment_id = experiment.id
@property
def remaining_observations(self):
"""
Re-fetch the experiment and calculate how many Observations we need to run until
we reach the observation budget
"""
experiment = self.conn.experiments(self.experiment_id).fetch()
return experiment.observation_budget - experiment.progress.observation_count
def run(self):
"""
Attempt to run NUM_WORKERS worker machines. If any machines fail, retry up to
three times, deleting openSuggestions before proceeding.
"""
tries = 3
while (tries > 0 and self.remaining_observations > 0):
workers = [Worker(self.experiment_id) for _ in xrange(NUM_WORKERS)]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
self.conn.experiments(
self.experiment_id).suggestions().delete(state='open')
tries -= 1
class Worker(threading.Thread):
'''worker thread which operates independently of all other workers and communicates only with sigopt server for
suggestions and sends observations'''
def __init__(self, evalfun, exptId, name, gpu):
threading.Thread.__init__(self)
self.conn = Connection(client_token=api_key)
self.exptId = exptId
self.name = name
self.gpu = gpu
self.evalfun = evalfun
@property
def metadata(self):
return dict(host=threading.current_thread().name)
@property
def remaining_observations(self):
experiment = self.conn.experiments(self.exptId).fetch()
return experiment.observation_budget - experiment.progress.observation_count
def run(self):
'''get suggetsions, run evaluation, report observation'''
# keep doing this loop till observations runs out
while self.remaining_observations > 0:
# get suggestion
suggestion = self.conn.experiments(
self.exptId).suggestions().create(metadata=self.metadata)
# run evaluation
try:
value = self.evalfun(assignment=suggestion.assignments,
gpu=self.gpu,
name=self.name + '-' + str(suggestion.id))
failed = False
except Exception:
value = None
failed = True
# report observation
self.conn.experiments(self.exptId).observations().create(
suggestion=suggestion.id,
value=value,
failed=failed,
metadata=self.metadata,
)
print(self.metadata,
'failed=' + str(failed)) # print for the console to see
def output_score(self, experiment, assignments, score, fout, sigopt_post=False):
"""Log the score, optionally save it to file, and/or report it back to SigOpt."""
suggestion = [assignments[param.name] for param in experiment.parameters]
output = "score: {suggestion} = {score}\n".format(suggestion=tuple(suggestion), score=score)
print(output, end='')
fout.write(output)
if sigopt_post is True:
conn = Connection(client_token=self.client_token)
conn.experiments(experiment.id).observations().create(
assignments=assignments,
value=score,
)
conn.experiments(experiment.id).suggestions().delete()
def __init__(self):
"""
Initialize the master thread, creating the SigOpt API connection and the experiment.
We use the observation_budget field on the experiment to keep track of approximately
how many total Observations we want to report. We recommend using a number between 10-20x
the number of parameters in an experiment.
"""
threading.Thread.__init__(self)
self.conn = Connection(client_token=SIGOPT_API_KEY)
experiment = self.conn.experiments().create(
name='Parallel Experiment',
parameters=PARAMETERS,
observation_budget=len(PARAMETERS) * 20,
)
print("View your experiment progress: https://sigopt.com/experiment/{}".format(experiment.id))
self.experiment_id = experiment.id
class Optimizer():
def __init__(self, experiment_id=None):
self.conn = Connection(client_token=SIG_KEY)
if experiment_id is None:
experiment = self.conn.experiments().create(
name='new_york_city',
parameters=PARAMS,
metrics=METRICS,
observation_budget=BUDGET,
project='pandemic')
self.experiment_id = experiment.id
else:
self.experiment_id = experiment_id
def optimize(self):
experiment = self.conn.experiments(self.experiment_id).fetch()
while experiment.progress.observation_count < experiment.observation_budget:
suggestion = self.conn.experiments(
experiment.id).suggestions().create()
value = evaluate_county(assignments=suggestion.assignments)
self.conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id,
value=value,
)
# Update the experiment object
experiment = self.conn.experiments(experiment.id).fetch()
# Fetch the best configuration and explore your experiment
all_best_assignments = self.conn.experiments(
experiment.id).best_assignments().fetch()
# Returns a list of dict-like Observation objects
best = all_best_assignments.data[0].assignments
return best
def _run_sig_opt(self, args):
"""
This function creates a new SigOpt experiment and optimizes the selected parameters.
SigOpt is a state-of-the-art Bayesian optimization tool. For more info on how to use
it see the tutorial page on: https://diva-dia.github.io/DeepDIVAweb/articles.html
Parameters
----------
args : dict
Contains all command line arguments parsed.
Returns
-------
None, None, None
At the moment it is not necessary to return meaningful values from here
"""
# Load parameters from file
with open(args.sig_opt, 'r') as f:
parameters = json.loads(f.read())
# Put your SigOpt token here.
if args.sig_opt_token is None:
logging.error('Enter your SigOpt API token using --sig-opt-token')
sys.exit(-1)
else:
conn = Connection(client_token=args.sig_opt_token)
experiment = conn.experiments().create(
name=args.experiment_name,
parameters=parameters,
observation_budget=args.sig_opt_runs,
project=args.sig_opt_project,
)
logging.info("Created experiment: https://sigopt.com/experiment/" +
experiment.id)
for i in range(args.sig_opt_runs):
# Get suggestion from SigOpt
suggestion = conn.experiments(
experiment.id).suggestions().create()
params = suggestion.assignments
for key in params:
if isinstance(args.__dict__[key], bool):
params[key] = params[key].lower() in ['true']
args.__dict__[key] = params[key]
_, _, score = self._execute(args)
# In case of multi-run the return type will be a list (otherwise is a single float)
if type(score) != float:
[
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id, value=item)
for item in score
]
else:
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id, value=score)
return None, None, None
def main(unused_args):
del unused_args
conn = Connection(client_token=FLAGS.api_token)
experiment = conn.experiments().create(
name=FLAGS.exp_name,
project=FLAGS.project_id,
metrics=[dict(name='pplx', objective='minimize')],
parameters=[
dict(name='mem_len', type='int', bounds=dict(min=0,
max=125)), # Multiply
#dict(name='perm_size', type='int', bounds=dict(min=1,max=2)), # Function
#dict(name='n_layer', type='int', bounds=dict(min=1,max=6)),
#dict(name='d_model', type='int', bounds=dict(min=5,max=10)), # Multiply
#dict(name='d_embed', type='int', bounds=dict(min=5,max=10)), # Multiply
#dict(name='n_head', type='int', bounds=dict(min=1,max=4)),
#dict(name='d_head', type='int', bounds=dict(min=1,max=6)), # Multiply
#dict(name='d_inner', type='int', bounds=dict(min=6,max=11)), # Multiply
dict(name='seq_len',
type='categorical',
categorical_values=['32', '64', '128', '256', '512']),
dict(name='learning_rate',
type='double',
bounds=dict(min=1e-8, max=1e-4)), # Multiply
dict(name='decay_method',
type='categorical',
categorical_values=['poly', 'cos']),
dict(name='dropout', type='int', bounds=dict(min=0,
max=5)), # Multiply
dict(name='dropatt', type='int', bounds=dict(min=0,
max=5)), # Multiply
dict(name='warmup_steps', type='int',
bounds=dict(min=0, max=10)), # Multiply
#dict(name='weight_decay', type='int', bounds=dict(min=-9, max=-4)) # Mutliply
],
observation_budget=FLAGS.budget,
parallel_bandwidth=FLAGS.num_workers,
)
logging.info("Experiment ID: " + str(experiment.id))
def run_hp_search_sigopt(trainer, n_trials: int, direction: str,
**kwargs) -> BestRun:
from sigopt import Connection
conn = Connection()
proxies = kwargs.pop("proxies", None)
if proxies is not None:
conn.set_proxies(proxies)
experiment = conn.experiments().create(
name="huggingface-tune",
parameters=trainer.hp_space(None),
metrics=[
dict(name="objective", objective=direction, strategy="optimize")
],
parallel_bandwidth=1,
observation_budget=n_trials,
project="huggingface",
)
logger.info(
f"created experiment: https://app.sigopt.com/experiment/{experiment.id}"
)
while experiment.progress.observation_count < experiment.observation_budget:
suggestion = conn.experiments(experiment.id).suggestions().create()
trainer.objective = None
trainer.train(resume_from_checkpoint=None, trial=suggestion)
# If there hasn't been any evaluation during the training loop.
if getattr(trainer, "objective", None) is None:
metrics = trainer.evaluate()
trainer.objective = trainer.compute_objective(metrics)
values = [dict(name="objective", value=trainer.objective)]
obs = conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id, values=values)
logger.info(
f"[suggestion_id, observation_id]: [{suggestion.id}, {obs.id}]")
experiment = conn.experiments(experiment.id).fetch()
best = list(
conn.experiments(
experiment.id).best_assignments().fetch().iterate_pages())[0]
best_run = BestRun(best.id, best.value, best.assignments)
return best_run
def create_experiment(self):
"""Create a SigOpt experiment for optimizing the classifier hyperparameters."""
conn = Connection(client_token=self.client_token)
params = CLASSIFIER_TYPE_TO_PARAMS[self.classifier_type]
try:
return conn.experiments().create(
name="Example Classifier",
parameters=params,
observation_budget=self.num_sigopt_suggestions,
)
except ApiException as err:
if err.status_code == 403 and '*****@*****.**' in str(err):
existing_experiments = list(conn.experiments().fetch().iterate_pages())
if existing_experiments:
raise Exception(
"You have existing experiments on sigopt.com: {0}."
" You have exceeded the number of experiments that can be created under your plan."
" Please visit https://sigopt.com/pricing to learn about plans."
.format(['https://sigopt.com/experiment/{0}'.format(e.id) for e in existing_experiments])
)
raise
def tune_sigopt(num_hidden, seed, experiment_name=None, experiment_id=None):
conn = Connection(client_token=SIGOPT_TOKEN)
if experiment_id is None:
# Create experiment
experiment = conn.experiments().create(
name=experiment_name,
parameters=[
dict(name='loglr', type='double', bounds=dict(min=LOGLR_MIN, max=LOGLR_MAX)),
],
)
print("Created experiment: https://sigopt.com/experiment/" + experiment.id)
experiment_id = experiment.id
else:
print("Continuing experiment: https://sigopt.com/experiment/" + experiment_id)
# Run the Optimization Loop between 10x - 20x the number of parameters
for _ in range(NUM_EVALS):
suggestion = conn.experiments(experiment_id).suggestions().create()
value = val_cost(suggestion.assignments['loglr'], num_hidden, seed, maximize=True)
print('Suggestion:', suggestion.assignments['loglr'], 'Validation value:', value)
conn.experiments(experiment_id).observations().create(
suggestion=suggestion.id,
value=value,
)
return experiment_id
def create_experiment(self):
"""Create a SigOpt experiment for optimizing the classifier hyperparameters."""
conn = Connection(client_token=self.client_token)
params = CLASSIFIER_TYPE_TO_PARAMS[self.classifier_type]
try:
return conn.experiments().create(
name="Example Classifier",
project="sigopt-examples",
parameters=params,
observation_budget=self.num_sigopt_suggestions,
)
except ApiException as err:
if err.status_code == 403 and '*****@*****.**' in str(err):
existing_experiments = list(conn.experiments().fetch().iterate_pages())
if existing_experiments:
raise Exception(
"You have existing experiments on sigopt.com: {0}."
" You have exceeded the number of experiments that can be created under your plan."
" Please visit https://sigopt.com/pricing to learn about plans."
.format(['https://sigopt.com/experiment/{0}'.format(e.id) for e in existing_experiments])
)
raise
def optimize(token, name, parameters, num_iter, echo=False):
conn = Connection(client_token=token)
experiment = conn.experiments().create(
name=name,
parameters=parameters,
)
def wrapper(func):
for ii in range(num_iter):
if echo:
print("*" * 60)
print("Running iteration #{}.".format(ii + 1))
suggestion = conn.experiments(experiment.id).suggestions().create()
if echo:
print("Suggested point: {}".format(suggestion.assignments))
value = func(suggestion.assignments)
if echo:
print("Value: {}".format(value))
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id, value=value)
result = conn.experiments(experiment.id).best_assignments().fetch()
if echo:
print("*" * 60)
for res in result.data:
print("Optimal value: {}".format(res.value))
print("Optimal point(s): {}".format(res.assignments))
print("*" * 60)
return result
return wrapper
def create_experiment_doc2vec(apikey):
conn = Connection(client_token=apikey)
experiment = conn.experiments().create(
name="Coronavirus Antivirals -- Doc2Vec",
parameters=[
dict(name="vec_dims", bounds=dict(min=300, max=512), type="int"),
dict(name="alpha", bounds=dict(min=0.02, max=0.1), type="double"),
dict(name="estimators", bounds=dict(min=350, max=512), type="int"),
dict(name="doc_epochs", bounds=dict(min=90, max=120), type="int"),
dict(name="vec_window", bounds=dict(min=2, max=4), type="int"),
dict(name="tree_criterion",
categorical_values=[dict(name="gini"),
dict(name="entropy")],
type="categorical")
],
metrics=[
dict(name="AUC", objective="maximize"),
dict(name="Model ID", strategy="store")
],
metadata=dict(template="antivirals"),
observation_budget=200,
parallel_bandwidth=12,
project="antivirals")
return experiment.id
def __init__(self, experiment_id=None, sigopt_config=None):
"""
Initiate a connection to the SigOpt API and optionally store the id
and config for an existing experiment. The SigOpt API key should be
defined in the environment variable 'SIGOPT_KEY'.
:param experiment_id: (int) An existing experiment id.
:param sigopt_config: (dict) The config used to create experiment id.
"""
self.experiment_id = experiment_id
self.sigopt_config = sigopt_config
self.conn = None
self.training_run = None
self.api_key = os.environ.get("SIGOPT_KEY", None)
if self.api_key is None:
self.api_key = os.environ.get("SIGOPT_DEV_KEY", None)
assert self.api_key is not None, "No SigOpt API key!"
try:
self.conn = Connection(client_token=self.api_key)
except Exception:
print("Could not connect to SigOpt!")
raise
def main():
hyper_params = {
"window_size":
10,
"layer_1":
16,
"layer_2":
4,
"features": [
"orientation_T8_q0",
"orientation_T8_q1",
"orientation_T8_q2",
"orientation_T8_q3",
],
"target":
"mode",
"epochs":
5
}
print("Loading dataset....")
df = pd.read_csv(f"../data/VTech/_VTech_merged.csv")
print_df_summary(df)
conn = Connection()
experiment = build_experiment(conn)
print(
f"Created experiment: https://app.sigopt.com/experiment/{experiment.id}"
)
for _ in range(experiment.observation_budget):
suggestion = conn.experiments(experiment.id).suggestions().create()
assignments = suggestion.assignments
hyper_params["window_size"] = assignments["window_size"]
hyper_params["layer_1"] = assignments["layer_1"]
hyper_params["layer_2"] = assignments["layer_2"]
print(f"Evaluating with: {hyper_params}")
ds = build_dataset(df, **hyper_params)
model = build_model(ds, **hyper_params)
accuracy = train_and_evaluate(model, ds, **hyper_params)
print(f"Accuracy: {accuracy}")
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id, value=accuracy)
assignments = conn.experiments(
experiment.id).best_assignments().fetch().data[0].assignments
print(assignments)
def main():
# Experimental Data
datafile = './data.csv'
data = importData(datafile)
# SBML model
modelfile = './simple.xml'
d = readSBML(modelfile)
# Simulation
simulation_time = 10
dt = 0.1
# Bayesian Optimization with SigOpt
param_id = 'k'
min_value = 0
max_value = 1.0
grid_spacing = 0.05
grid_points = (max_value - min_value) / grid_spacing + 1
## Set up the experiment
conn = Connection(client_token=my_token)
experiment = conn.experiments().create(
name=d.getModel().getId() + " SBML model",
parameters=[
dict(name=param_id,
type='double',
bounds=dict(min=min_value, max=max_value)),
],
)
print("Created experiment: https://sigopt.com/experiment/" + experiment.id)
# Run the Optimization Loop between 10x - 20x the number of parameters
for _ in range(20):
suggestion = conn.experiments(experiment.id).suggestions().create()
print _, "suggestion:", param_id, "=", suggestion.assignments[param_id]
updateParameter(d, param_id, suggestion.assignments[param_id])
result = integrate(d, simulation_time, dt)
value = calcError(result, data) * -1
print " value =", value
#value = evaluate_model(suggestion.assignments)
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id,
value=value,
)
best_assignments_list = (conn.experiments(
experiment.id).best_assignments().fetch())
if best_assignments_list.data:
#print best_assignments_list.data
best_assignments = best_assignments_list.data[0].assignments
print best_assignments
def sigopt_generator(self, experiment):
"""Generate optimal parameter configurations using SigOpt."""
for _ in xrange(experiment.observation_budget):
conn = Connection(client_token=self.client_token)
suggestion = conn.experiments(experiment.id).suggestions().create()
yield suggestion.assignments.to_json()
def __init__(self, client_token, experiment_id): threading.Thread.__init__(self) self.connection = Connection(client_token=client_token) self.experiment_id = experiment_id
import argparse
from sigopt import Connection
# Take a suggestion from sigopt and evaluate your function
def evaluate_metric(assignments):
x = assignments['x']
raise NotImplementedError("Return a number, which represents your metric evaluated at x")
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--observation_budget', type=int, default=20)
parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens")
the_args = parser.parse_args()
connection = Connection(client_token=the_args.client_token)
# Create an experiment with one paramter, x
experiment = connection.experiments().create(
name="Basic Test experiment",
project="sigopt-examples",
parameters=[{'name': 'x', 'bounds': {'max': 50.0, 'min': 0.0}, 'type': 'double'}],
observation_budget=the_args.observation_budget,
)
print('Created experiment id {0}'.format(experiment.id))
# In a loop: receive a suggestion, evaluate the metric, report an observation
for _ in range(experiment.observation_budget):
suggestion = connection.experiments(experiment.id).suggestions().create()
print('Evaluating at suggested assignments: {0}'.format(suggestion.assignments))
value = evaluate_metric(suggestion.assignments)
Replace this with the function you want to optimize
This fictitious example has only two parameters, named x1 and x2
"""
sleep_seconds = 10
print('{0} - Sleeping for {1} seconds to simulate expensive computation...'.format(threading.current_thread(), sleep_seconds))
time.sleep(sleep_seconds)
return sigopt.examples.franke_function(assignments['x1'], assignments['x2'])
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--runner_count', type=int, default=2)
parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens")
the_args = parser.parse_args()
client_token = the_args.client_token
conn = Connection(client_token=client_token)
experiment = conn.experiments().create(
name="Parallel Test Franke Function",
project="sigopt-examples",
parameters=[
{'name': 'x1', 'bounds': {'max': 1.0, 'min': 0.0}, 'type': 'double'},
{'name': 'x2', 'bounds': {'max': 1.0, 'min': 0.0}, 'type': 'double'},
],
)
runners = [ExampleRunner(client_token, experiment.id) for _ in range(the_args.runner_count)]
for runner in runners:
runner.daemon = True
runner.start()
(stdoutdata,stderrdata) = process.communicate()
sys.stderr.write(stderrdata)
return float(stdoutdata.strip())
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--command', required=True, help="The command to run the function whose parameters you would "
"like to optimize. Should accept parameters as command line argument and output only the evaluated metric at the "
"suggested point.")
parser.add_argument('--experiment_id', required=True, help="The parameters of this experiment should be the "
"same type and name of the command line arguments to your executable file.")
parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens")
the_args = parser.parse_args()
connection = Connection(client_token=the_args.client_token)
experiment = connection.experiments(the_args.experiment_id).fetch()
connection.experiments(the_args.experiment_id).suggestions().delete(state="open")
evaluator = SubProcessEvaluator(the_args.command)
# In a loop: receive a suggestion, evaluate the metric, report an observation
while True:
suggestion = connection.experiments(experiment.id).suggestions().create()
print('Evaluating at suggested assignments: {0}'.format(suggestion.assignments))
value = evaluator.evaluate_metric(suggestion.assignments)
print('Reporting observation of value: {0}'.format(value))
connection.experiments(experiment.id).observations().create(
suggestion=suggestion.id,
value=value,
)
# Use SigOpt to tune a Random Forest Classifier in Python
# Learn more about SigOpt's Python Client:
# https://sigopt.com/docs/overview/python
# Run `pip install sigopt` to download the python API client
# Run `pip install sklearn` to install scikit-learn, a machine learning
# library in Python (http://scikit-learn.org)
from sigopt import Connection
from sklearn import cross_validation, datasets
from sklearn.ensemble import RandomForestClassifier
import numpy
# Learn more about authenticating the SigOpt API:
# https://sigopt.com/docs/overview/authentication
conn = Connection(client_token=SIGOPT_API_TOKEN)
# Load dataset
# We are using the iris dataset as an example
iris = datasets.load_iris()
X = iris.data
y = iris.target
# Create a SigOpt experiment for the Random Forest parameters
experiment = conn.experiments().create(
name="Random Forest (Python)",
parameters=[
dict(name="max_features", type="int", bounds=dict(min=1, max=len(iris)-1)),
dict(name="n_estimators", type="int", bounds=dict(min=1, max=100)),
dict(name="min_samples_leaf", type="int", bounds=dict(min=1, max=10))
]
)
