class ExampleRunner(threading.Thread): def __init__(self, client_token, experiment_id): threading.Thread.__init__(self) self.connection = Connection(client_token=client_token) self.experiment_id = experiment_id def run(self): while True: suggestion = self.connection.experiments(self.experiment_id).suggestions().create() print('{0} - Evaluating at parameters: {1}'.format(threading.current_thread(), suggestion.assignments)) value = self.evaluate_metric(suggestion.assignments) print('{0} - Observed value: {1}'.format(threading.current_thread(), value)) self.connection.experiments(self.experiment_id).observations().create( suggestion=suggestion.id, value=value, ) def evaluate_metric(self, assignments): """ Replace this with the function you want to optimize This fictitious example has only two parameters, named x1 and x2 """ sleep_seconds = 10 print('{0} - Sleeping for {1} seconds to simulate expensive computation...'.format(threading.current_thread(), sleep_seconds)) time.sleep(sleep_seconds) return sigopt.examples.franke_function(assignments['x1'], assignments['x2'])
def run_sigopt_optimization(): conn = Connection(client_token=os.environ['SIGOPT_TOKEN']) # experiment = conn.experiments().create( # name='Simple CNN Sentiment Analysis Hyperparameter Search', # parameters=[ # dict(name='top_words', type='int', bounds=dict(min=100, max=88000)), # dict(name='max_words', type='int', bounds=dict(min=10, max=1000)), # dict(name='filters', type='int', bounds=dict(min=0, max=10)), # dict(name='kernel_size', type='int', bounds=dict(min=1, max=5)), # dict(name='pool_size', type='int', bounds=dict(min=2, max=8)), # dict(name='dense_size', type='int', bounds=dict(min=2, max=1024)), # dict(name='batch_size', type='int', bounds=dict(min=2, max=1024)), # ], # ) # print("Created experiment: https://sigopt.com/experiment/" + experiment.id) e_id = str(48737) for _ in range(300): suggestion = conn.experiments(e_id).suggestions().create() options = suggestion.assignments s = SentimentAnalyzer(options) s.train_classifier() value = s.evaluate_classifier() conn.experiments(e_id).observations().create( suggestion=suggestion.id, value=value[1], )
def tune_sigopt(num_hidden, seed, experiment_name=None, experiment_id=None): conn = Connection(client_token=SIGOPT_TOKEN) if experiment_id is None: # Create experiment experiment = conn.experiments().create( name=experiment_name, parameters=[ dict(name='loglr', type='double', bounds=dict(min=LOGLR_MIN, max=LOGLR_MAX)), ], ) print("Created experiment: https://sigopt.com/experiment/" + experiment.id) experiment_id = experiment.id else: print("Continuing experiment: https://sigopt.com/experiment/" + experiment_id) # Run the Optimization Loop between 10x - 20x the number of parameters for _ in range(NUM_EVALS): suggestion = conn.experiments(experiment_id).suggestions().create() value = val_cost(suggestion.assignments['loglr'], num_hidden, seed, maximize=True) print('Suggestion:', suggestion.assignments['loglr'], 'Validation value:', value) conn.experiments(experiment_id).observations().create( suggestion=suggestion.id, value=value, ) return experiment_id
def _run_sig_opt(self, args): """ This function creates a new SigOpt experiment and optimizes the selected parameters. SigOpt is a state-of-the-art Bayesian optimization tool. For more info on how to use it see the tutorial page on: https://diva-dia.github.io/DeepDIVAweb/articles.html Parameters ---------- args : dict Contains all command line arguments parsed. Returns ------- None, None, None At the moment it is not necessary to return meaningful values from here """ # Load parameters from file with open(args.sig_opt, 'r') as f: parameters = json.loads(f.read()) # Put your SigOpt token here. if args.sig_opt_token is None: logging.error('Enter your SigOpt API token using --sig-opt-token') sys.exit(-1) else: conn = Connection(client_token=args.sig_opt_token) experiment = conn.experiments().create( name=args.experiment_name, parameters=parameters, observation_budget=args.sig_opt_runs, project=args.sig_opt_project, ) logging.info("Created experiment: https://sigopt.com/experiment/" + experiment.id) for i in range(args.sig_opt_runs): # Get suggestion from SigOpt suggestion = conn.experiments( experiment.id).suggestions().create() params = suggestion.assignments for key in params: if isinstance(args.__dict__[key], bool): params[key] = params[key].lower() in ['true'] args.__dict__[key] = params[key] _, _, score = self._execute(args) # In case of multi-run the return type will be a list (otherwise is a single float) if type(score) != float: [ conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=item) for item in score ] else: conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=score) return None, None, None
class Master(threading.Thread): '''master thread which initializes an experimetn and starts the workers (passing in the expt id and gpu it should run on. doesnt seem necessary for this to be its own class nor does it seem necessary for it to inhereit the threading class.''' def __init__(self, evalfun, exptDetail, name, gpus, exptId=None, resume=False, **kwargs): threading.Thread.__init__( self) # required: call constructor of threading class self.evalfun = evalfun self.gpus = gpus self.conn = Connection( client_token=api_key) # start connection to sigopt if exptId == None: # start anew experiment expt = self.conn.experiments().create(**exptDetail) exptId = expt.id else: # use existing experiment id expt = self.conn.experiments(exptId) expt.update(**exptDetail) if not resume: # delete all observations and suggestions expt.observations().delete() expt.suggestions().delete() self.name = name self.exptId = exptId # save expt id in the instance print( "View your experiment progress: https://sigopt.com/experiment/{}". format(self.exptId)) @property def remaining_observations(self): experiment = self.conn.experiments(self.exptId).fetch() return experiment.observation_budget - experiment.progress.observation_count def run(self): '''start the workers and evaluations''' tries = 3 while (tries > 0 and self.remaining_observations > 0): workers = [ Worker(self.evalfun, exptId=self.exptId, name=self.name, gpu=gpu) for gpu in self.gpus ] for worker in workers: worker.start( ) # each worker will communicated spearately with sigopt and will not communicate wiht one another for worker in workers: worker.join( ) # wait till all workers are done, which only happens when observation budget runs out self.conn.experiments( self.exptId).suggestions().delete(state='open') tries -= 1 # will reach this point only when observation budget runs out or all workers have failed
class Worker(threading.Thread): """ Shows what a worker machine does when running SigOpt in a distributed setting. """ def __init__(self, experiment_id): """ Initialize a worker thread, creating the SigOpt API connection and storing the previously created experiment's id """ threading.Thread.__init__(self) self.experiment_id = experiment_id self.conn = Connection(client_token=SIGOPT_API_KEY) @property def metadata(self): """ Use metadata to keep track of the host that each Suggestion and Observation is created on. Learn more: https://sigopt.com/docs/overview/metadata """ return dict(host=threading.current_thread().name) @property def remaining_observations(self): """ Re-fetch the experiment and calculate how many Observations we need to run until we reach the observation budget """ experiment = self.conn.experiments(self.experiment_id).fetch() return experiment.observation_budget - experiment.progress.observation_count def run(self): """ SigOpt acts as the scheduler for the Suggestions, so all you need to do is run the optimization loop until there are no remaining Observations to be reported. We handle exceptions by reporting failed Observations. Learn more about handling failure cases: https://sigopt.com/docs/overview/metric_failure """ while self.remaining_observations > 0: suggestion = self.conn.experiments( self.experiment_id).suggestions().create( metadata=self.metadata) try: value = evaluate_model(suggestion.assignments) failed = False except Exception: value = None failed = True self.conn.experiments(self.experiment_id).observations().create( suggestion=suggestion.id, value=value, failed=failed, metadata=self.metadata, )
class SigOpt(AbstractSolver): def __init__(self, params): super().__init__(params) self.experiment_id = os.getenv("SIGOPT_EXPERIMENT", None) api_token = os.getenv("SIGOPT_API_TOKEN") if api_token is None: raise ValueError( "envvar SIGOPT_API_TOKEN must be set with SigOptSolver") self.conn = Connection(client_token=api_token) self.logger = logging.getLogger( ".".join(__name__.split(".")[:-1] + ["SigOptSolver"])) if self.experiment_id is None: params = flatten_params(params) experiment = self.conn.experiments().create( name=f'Mincall opt {socket.gethostname()}', parameters=[{ "name": name, "type": p.type, "bounds": { "min": p.min, "max": p.max } } for name, p in params.items()], ) self.experiment_id = experiment.id self.logger.info( f"Created experiment: https://sigopt.com/experiment/{self.experiment_id} in " f"{'DEVELOPMENT' if experiment.development else 'PRODUCTION'} mode" ) else: self.logger.info( "Using experiment: https://sigopt.com/experiment/" + self.experiment_id) def new_assignment(self): suggestion = self.conn.experiments( self.experiment_id).suggestions().create() return Assignment( params=unflatten_params(suggestion.assignments), name=f"{suggestion.id}-{name_generator()}", context=suggestion.id, ) def report(self, assignment: Assignment, observation: Observation): self.logger.info( f"Assignment:\n{pformat(dict(assignment._asdict()))}\n" f"has observation:\n{pformat(dict(observation._asdict()))}") self.conn.experiments(self.experiment_id).observations().create( suggestion=assignment.context, value=observation.metric, value_stddev=observation.metric_std, metadata=toolz.merge(observation.metadata, {'model_name': assignment.name}))
def main(): hyper_params = { "window_size": 10, "layer_1": 16, "layer_2": 4, "features": [ "orientation_T8_q0", "orientation_T8_q1", "orientation_T8_q2", "orientation_T8_q3", ], "target": "mode", "epochs": 5 } print("Loading dataset....") df = pd.read_csv(f"../data/VTech/_VTech_merged.csv") print_df_summary(df) conn = Connection() experiment = build_experiment(conn) print( f"Created experiment: https://app.sigopt.com/experiment/{experiment.id}" ) for _ in range(experiment.observation_budget): suggestion = conn.experiments(experiment.id).suggestions().create() assignments = suggestion.assignments hyper_params["window_size"] = assignments["window_size"] hyper_params["layer_1"] = assignments["layer_1"] hyper_params["layer_2"] = assignments["layer_2"] print(f"Evaluating with: {hyper_params}") ds = build_dataset(df, **hyper_params) model = build_model(ds, **hyper_params) accuracy = train_and_evaluate(model, ds, **hyper_params) print(f"Accuracy: {accuracy}") conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=accuracy) assignments = conn.experiments( experiment.id).best_assignments().fetch().data[0].assignments print(assignments)
class Master(threading.Thread): """ Shows what a master machine does when running SigOpt in a distributed setting. """ def __init__(self): """ Initialize the master thread, creating the SigOpt API connection and the experiment. We use the observation_budget field on the experiment to keep track of approximately how many total Observations we want to report. We recommend using a number between 10-20x the number of parameters in an experiment. """ threading.Thread.__init__(self) self.conn = Connection(client_token=SIGOPT_API_KEY) experiment = self.conn.experiments().create( name='Parallel Experiment', project='sigopt-examples', parameters=PARAMETERS, metrics=[dict(name='cv_accuracies', objective='maximize')], observation_budget=len(PARAMETERS) * 20, ) print( "View your experiment progress: https://sigopt.com/experiment/{}". format(experiment.id)) self.experiment_id = experiment.id @property def remaining_observations(self): """ Re-fetch the experiment and calculate how many Observations we need to run until we reach the observation budget """ experiment = self.conn.experiments(self.experiment_id).fetch() return experiment.observation_budget - experiment.progress.observation_count def run(self): """ Attempt to run NUM_WORKERS worker machines. If any machines fail, retry up to three times, deleting openSuggestions before proceeding. """ tries = 3 while (tries > 0 and self.remaining_observations > 0): workers = [Worker(self.experiment_id) for _ in xrange(NUM_WORKERS)] for worker in workers: worker.start() for worker in workers: worker.join() self.conn.experiments( self.experiment_id).suggestions().delete(state='open') tries -= 1
class Worker(threading.Thread): '''worker thread which operates independently of all other workers and communicates only with sigopt server for suggestions and sends observations''' def __init__(self, evalfun, exptId, name, gpu): threading.Thread.__init__(self) self.conn = Connection(client_token=api_key) self.exptId = exptId self.name = name self.gpu = gpu self.evalfun = evalfun @property def metadata(self): return dict(host=threading.current_thread().name) @property def remaining_observations(self): experiment = self.conn.experiments(self.exptId).fetch() return experiment.observation_budget - experiment.progress.observation_count def run(self): '''get suggetsions, run evaluation, report observation''' # keep doing this loop till observations runs out while self.remaining_observations > 0: # get suggestion suggestion = self.conn.experiments( self.exptId).suggestions().create(metadata=self.metadata) # run evaluation try: value = self.evalfun(assignment=suggestion.assignments, gpu=self.gpu, name=self.name + '-' + str(suggestion.id)) failed = False except Exception: value = None failed = True # report observation self.conn.experiments(self.exptId).observations().create( suggestion=suggestion.id, value=value, failed=failed, metadata=self.metadata, ) print(self.metadata, 'failed=' + str(failed)) # print for the console to see
def output_score(self, experiment, assignments, score, fout, sigopt_post=False): """Log the score, optionally save it to file, and/or report it back to SigOpt.""" suggestion = [assignments[param.name] for param in experiment.parameters] output = "score: {suggestion} = {score}\n".format(suggestion=tuple(suggestion), score=score) print(output, end='') fout.write(output) if sigopt_post is True: conn = Connection(client_token=self.client_token) conn.experiments(experiment.id).observations().create( assignments=assignments, value=score, ) conn.experiments(experiment.id).suggestions().delete()
def create_experiment_lda(apikey): conn = Connection(client_token=apikey) experiment = conn.experiments().create( name="Coronavirus Antivirals -- LDA", parameters=[ dict(name="topics", bounds=dict(min=16, max=512), type="int"), dict(name="estimators", bounds=dict(min=256, max=512), type="int"), dict(name="topic_epochs", bounds=dict(min=1, max=6), type="int"), dict(name="decay", bounds=dict(min=0.5, max=0.999), type="double"), dict(name="max_vocab", bounds=dict(min=10000, max=100000), type="int"), dict(name="min_samples_split", bounds=dict(min=4, max=18), type="int"), dict(name="min_samples_leaf", bounds=dict(min=2, max=4), type="int"), dict(name="topic_iterations", bounds=dict(min=20, max=200), type="int"), dict(name="tree_criterion", categorical_values=[dict(name="gini"), dict(name="entropy")], type="categorical") ], metrics=[ dict(name="AUC", objective="maximize"), dict(name="Model ID", strategy="store") ], metadata=dict(template="antivirals"), observation_budget=300, parallel_bandwidth=10, project="antivirals") return experiment.id
def create_experiment(apikey): conn = Connection(client_token=apikey) experiment = conn.experiments().create( name="Coronaboard -- AutoARIMA", parameters=[ dict(name="max_p", bounds=dict(min=14, max=30), type="int"), dict(name="max_q", bounds=dict(min=14, max=30), type="int"), dict(name="max_d", bounds=dict(min=14, max=30), type="int"), dict(name="maxiter", bounds=dict(min=80, max=360), type="int"), dict(name="scoring", categorical_values=[dict(name="mse"), dict(name="mae")], type="categorical"), dict(name="information_criterion", categorical_values=[ dict(name="aic"), dict(name="bic"), dict(name="hqic"), dict(name="oob") ], type="categorical") ], metrics=[dict(name="SMAPE", objective="minimize")], metadata=dict(template="dashboard"), observation_budget=400, parallel_bandwidth=12, project="dashboard-dev") return experiment.id
def main(): # Experimental Data datafile = './data.csv' data = importData(datafile) # SBML model modelfile = './simple.xml' d = readSBML(modelfile) # Simulation simulation_time = 10 dt = 0.1 # Bayesian Optimization with SigOpt param_id = 'k' min_value = 0 max_value = 1.0 grid_spacing = 0.05 grid_points = (max_value - min_value) / grid_spacing + 1 ## Set up the experiment conn = Connection(client_token=my_token) experiment = conn.experiments().create( name=d.getModel().getId() + " SBML model", parameters=[ dict(name=param_id, type='double', bounds=dict(min=min_value, max=max_value)), ], ) print("Created experiment: https://sigopt.com/experiment/" + experiment.id) # Run the Optimization Loop between 10x - 20x the number of parameters for _ in range(20): suggestion = conn.experiments(experiment.id).suggestions().create() print _, "suggestion:", param_id, "=", suggestion.assignments[param_id] updateParameter(d, param_id, suggestion.assignments[param_id]) result = integrate(d, simulation_time, dt) value = calcError(result, data) * -1 print " value =", value #value = evaluate_model(suggestion.assignments) conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value, ) best_assignments_list = (conn.experiments( experiment.id).best_assignments().fetch()) if best_assignments_list.data: #print best_assignments_list.data best_assignments = best_assignments_list.data[0].assignments print best_assignments
def _run_sig_opt(self, args): # TODO: improve doc """ This function creates a new SigOpt experiment and optimizes the selected parameters. Parameters: ----------- :param args: :return: None """ # Load parameters from file with open(args.sig_opt, 'r') as f: parameters = json.loads(f.read()) # Put your SigOpt token here. if args.sig_opt_token == None: print('Enter your SigOpt API token using --sig-opt-token') sys.exit(0) else: conn = Connection(client_token=args.sig_opt_token) experiment = conn.experiments().create( name=args.experiment_name, parameters=parameters, ) print("Created experiment: https://sigopt.com/experiment/" + experiment.id) for i in range(args.sig_opt_runs): # Get suggestion from SigOpt suggestion = conn.experiments( experiment.id).suggestions().create() params = suggestion.assignments for key in params: if isinstance(args.__dict__[key], bool): params[key] = params[key].lower() in ['true'] args.__dict__[key] = params[key] _, _, score = self._execute(args) # In case of multi-run the return type will be a list (otherwise is a single float) if type(score) != float: [ conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=item) for item in score ] else: conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=score)
def run_hp_search_sigopt(trainer, n_trials: int, direction: str, **kwargs) -> BestRun: from sigopt import Connection conn = Connection() proxies = kwargs.pop("proxies", None) if proxies is not None: conn.set_proxies(proxies) experiment = conn.experiments().create( name="huggingface-tune", parameters=trainer.hp_space(None), metrics=[ dict(name="objective", objective=direction, strategy="optimize") ], parallel_bandwidth=1, observation_budget=n_trials, project="huggingface", ) logger.info( f"created experiment: https://app.sigopt.com/experiment/{experiment.id}" ) while experiment.progress.observation_count < experiment.observation_budget: suggestion = conn.experiments(experiment.id).suggestions().create() trainer.objective = None trainer.train(resume_from_checkpoint=None, trial=suggestion) # If there hasn't been any evaluation during the training loop. if getattr(trainer, "objective", None) is None: metrics = trainer.evaluate() trainer.objective = trainer.compute_objective(metrics) values = [dict(name="objective", value=trainer.objective)] obs = conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, values=values) logger.info( f"[suggestion_id, observation_id]: [{suggestion.id}, {obs.id}]") experiment = conn.experiments(experiment.id).fetch() best = list( conn.experiments( experiment.id).best_assignments().fetch().iterate_pages())[0] best_run = BestRun(best.id, best.value, best.assignments) return best_run
def continue_experiment(apikey, exp_id): counties_url = 'https://github.com/nytimes/covid-19-data/raw/42378bc95a04ff4fe798d2378affb351954db164/us-counties.csv' us_counties = get_counties_data(counties_url) conn = Connection(client_token=apikey) experiment = conn.experiments(exp_id).fetch() for _ in range(experiment.observation_budget): try: suggestion = conn.experiments(exp_id).suggestions().create() except ApiException: conn.experiments(exp_id).suggestions().delete() suggestion = conn.experiments(exp_id).suggestions().create() assignments = suggestion.assignments print(f"Hyperpameters: {assignments}") try: _, _, metrics = forecast(us_counties, log_metrics=True, hp={ **HYPERPARAMETERS, **assignments }) mean = sum(metrics.values()) / len(metrics) except Exception as e: print("Invalid hyperparameter combination!") print(str(e)) mean = 0.9999 finally: conn.experiments(exp_id).observations().create( suggestion=suggestion.id, value=mean)
class Optimizer(): def __init__(self, experiment_id=None): self.conn = Connection(client_token=SIG_KEY) if experiment_id is None: experiment = self.conn.experiments().create( name='new_york_city', parameters=PARAMS, metrics=METRICS, observation_budget=BUDGET, project='pandemic') self.experiment_id = experiment.id else: self.experiment_id = experiment_id def optimize(self): experiment = self.conn.experiments(self.experiment_id).fetch() while experiment.progress.observation_count < experiment.observation_budget: suggestion = self.conn.experiments( experiment.id).suggestions().create() value = evaluate_county(assignments=suggestion.assignments) self.conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value, ) # Update the experiment object experiment = self.conn.experiments(experiment.id).fetch() # Fetch the best configuration and explore your experiment all_best_assignments = self.conn.experiments( experiment.id).best_assignments().fetch() # Returns a list of dict-like Observation objects best = all_best_assignments.data[0].assignments return best
def continue_experiment(dbstring, apikey, exp_id): conn = Connection(client_token=apikey) experiment = conn.experiments(exp_id).fetch() for _ in range(experiment.observation_budget): try: suggestion = conn.experiments(exp_id).suggestions().create() except ApiException: conn.experiments(exp_id).suggestions().delete() suggestion = conn.experiments(exp_id).suggestions().create() if suggestion.assignments.get('topics'): suggestion.assignments['vector_algo'] = 'lda' elif suggestion.assignments.get('vec_dims'): suggestion.assignments['vector_algo'] = 'doc2vec' assignments = Hyperparameters.from_dict(suggestion.assignments) chem = run_train_models(dbstring, assignments) mean = sum(chem.toxicity.auc) / len(chem.toxicity.auc) conn.experiments(exp_id).observations().create( suggestion=suggestion.id, values=[ dict( name="AUC", value=mean, ), dict(name="Model ID", value=UUID(chem.uuid).int) ])
def create_experiment(self): """Create a SigOpt experiment for optimizing the classifier hyperparameters.""" conn = Connection(client_token=self.client_token) params = CLASSIFIER_TYPE_TO_PARAMS[self.classifier_type] try: return conn.experiments().create( name="Example Classifier", parameters=params, observation_budget=self.num_sigopt_suggestions, ) except ApiException as err: if err.status_code == 403 and '*****@*****.**' in str(err): existing_experiments = list(conn.experiments().fetch().iterate_pages()) if existing_experiments: raise Exception( "You have existing experiments on sigopt.com: {0}." " You have exceeded the number of experiments that can be created under your plan." " Please visit https://sigopt.com/pricing to learn about plans." .format(['https://sigopt.com/experiment/{0}'.format(e.id) for e in existing_experiments]) ) raise
def sigopt_cube(objective, scale, n_trials): """ :param objective: returns 1-tuple :param scale: float defines cubical domain :param n_trials: int :return: """ params = [{'name': 'u1', 'type': 'double', 'bounds': {'min': -scale, 'max': scale}}, {'name': 'u2', 'type': 'double', 'bounds': {'min': -scale, 'max': scale}}, {'name': 'u3', 'type': 'double', 'bounds': {'min': -scale, 'max': scale}}] metrics = [{'name': 'slimy_moose', 'objective': 'minimize'}] conn = Connection(client_token=SIG_KEY) experiment = conn.experiments().create(name='exp_'+str(uuid.uuid4())[:6], parameters=params, metrics=metrics, observation_budget=n_trials, project=PROJECT) def _objective(assignments): u = [assignments['u1'], assignments['u2'], assignments['u3']] return objective(u)[0] while experiment.progress.observation_count < experiment.observation_budget: suggestion = conn.experiments(experiment.id).suggestions().create() value = _objective(assignments=suggestion.assignments) conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value, ) experiment = conn.experiments(experiment.id).fetch() all_best_assignments = conn.experiments(experiment.id).best_assignments().fetch() return all_best_assignments.data[0].value
def create_experiment(self): """Create a SigOpt experiment for optimizing the classifier hyperparameters.""" conn = Connection(client_token=self.client_token) params = CLASSIFIER_TYPE_TO_PARAMS[self.classifier_type] try: return conn.experiments().create( name="Example Classifier", project="sigopt-examples", parameters=params, observation_budget=self.num_sigopt_suggestions, ) except ApiException as err: if err.status_code == 403 and '*****@*****.**' in str(err): existing_experiments = list(conn.experiments().fetch().iterate_pages()) if existing_experiments: raise Exception( "You have existing experiments on sigopt.com: {0}." " You have exceeded the number of experiments that can be created under your plan." " Please visit https://sigopt.com/pricing to learn about plans." .format(['https://sigopt.com/experiment/{0}'.format(e.id) for e in existing_experiments]) ) raise
def main(unused_args): del unused_args conn = Connection(client_token=FLAGS.api_token) experiment = conn.experiments().create( name=FLAGS.exp_name, project=FLAGS.project_id, metrics=[dict(name='pplx', objective='minimize')], parameters=[ dict(name='mem_len', type='int', bounds=dict(min=0, max=125)), # Multiply #dict(name='perm_size', type='int', bounds=dict(min=1,max=2)), # Function #dict(name='n_layer', type='int', bounds=dict(min=1,max=6)), #dict(name='d_model', type='int', bounds=dict(min=5,max=10)), # Multiply #dict(name='d_embed', type='int', bounds=dict(min=5,max=10)), # Multiply #dict(name='n_head', type='int', bounds=dict(min=1,max=4)), #dict(name='d_head', type='int', bounds=dict(min=1,max=6)), # Multiply #dict(name='d_inner', type='int', bounds=dict(min=6,max=11)), # Multiply dict(name='seq_len', type='categorical', categorical_values=['32', '64', '128', '256', '512']), dict(name='learning_rate', type='double', bounds=dict(min=1e-8, max=1e-4)), # Multiply dict(name='decay_method', type='categorical', categorical_values=['poly', 'cos']), dict(name='dropout', type='int', bounds=dict(min=0, max=5)), # Multiply dict(name='dropatt', type='int', bounds=dict(min=0, max=5)), # Multiply dict(name='warmup_steps', type='int', bounds=dict(min=0, max=10)), # Multiply #dict(name='weight_decay', type='int', bounds=dict(min=-9, max=-4)) # Mutliply ], observation_budget=FLAGS.budget, parallel_bandwidth=FLAGS.num_workers, ) logging.info("Experiment ID: " + str(experiment.id))
def optimize(token, name, parameters, num_iter, echo=False): conn = Connection(client_token=token) experiment = conn.experiments().create( name=name, parameters=parameters, ) def wrapper(func): for ii in range(num_iter): if echo: print("*" * 60) print("Running iteration #{}.".format(ii + 1)) suggestion = conn.experiments(experiment.id).suggestions().create() if echo: print("Suggested point: {}".format(suggestion.assignments)) value = func(suggestion.assignments) if echo: print("Value: {}".format(value)) conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value) result = conn.experiments(experiment.id).best_assignments().fetch() if echo: print("*" * 60) for res in result.data: print("Optimal value: {}".format(res.value)) print("Optimal point(s): {}".format(res.assignments)) print("*" * 60) return result return wrapper
def create_experiment_doc2vec(apikey): conn = Connection(client_token=apikey) experiment = conn.experiments().create( name="Coronavirus Antivirals -- Doc2Vec", parameters=[ dict(name="vec_dims", bounds=dict(min=300, max=512), type="int"), dict(name="alpha", bounds=dict(min=0.02, max=0.1), type="double"), dict(name="estimators", bounds=dict(min=350, max=512), type="int"), dict(name="doc_epochs", bounds=dict(min=90, max=120), type="int"), dict(name="vec_window", bounds=dict(min=2, max=4), type="int"), dict(name="tree_criterion", categorical_values=[dict(name="gini"), dict(name="entropy")], type="categorical") ], metrics=[ dict(name="AUC", objective="maximize"), dict(name="Model ID", strategy="store") ], metadata=dict(template="antivirals"), observation_budget=200, parallel_bandwidth=12, project="antivirals") return experiment.id
def sigopt_cube(objective, scale, n_trials, n_dim, with_count=False): """ :param objective: returns 1-tuple :param scale: float defines cubical domain :param n_trials: int :return: """ params = [ {'name':'u'+str(i),'type':'double', 'bounds': {'min': -scale, 'max': scale}} for i in range(n_dim) ] metrics = [{'name': 'slimy_moose', 'objective': 'minimize'}] conn = Connection(client_token=SIG_KEY) experiment = conn.experiments().create(name='exp_'+str(uuid.uuid4())[:8], parameters=params, metrics=metrics, observation_budget=n_trials, project=PROJECT) def _objective(assignments): u = [assignments['u'+str(i)] for i in range(n_dim) ] return objective(u)[0] while experiment.progress.observation_count < experiment.observation_budget: suggestion = conn.experiments(experiment.id).suggestions().create() value = _objective(assignments=suggestion.assignments) conn.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value, ) experiment = conn.experiments(experiment.id).fetch() got_results = False for _ in range(5): if not got_results: try: all_best_assignments = conn.experiments(experiment.id).best_assignments().fetch() got_results = True except: time.sleep(60) return (all_best_assignments.data[0].value, n_trials) if with_count else all_best_assignments.data[0].value
def main(_): conn = Connection(client_token=FLAGS.api_token) observations = conn.experiments( FLAGS.experiment_id).observations().fetch(state="failed") for obs in observations.iterate_pages(): suggestion_id = str(obs.suggestion) model_dir = os.path.join(FLAGS.bucket_name, "models", suggestion_id) param_config_file = os.path.join(FLAGS.bucket_name, "param_configs", "{}.json".format(suggestion_id)) if tf.io.gfile.exists(model_dir): tf.gfile.DeleteRecursively(model_dir) if tf.io.gfile.exists(param_config_file): tf.gfile.Remove(param_config_file) conn.experiments(FLAGS.experiment_id).observations().delete(state="failed") if FLAGS.total: suggestions = conn.experiments( FLAGS.experiment_id).suggestions().fetch(state="open") for suggestion in suggestions.iterate_pages(): model_dir = os.path.join(FLAGS.bucket_name, "models", suggestion.id) param_config_file = os.path.join(FLAGS.bucket_name, "param_configs", "{}.json".format(suggestion.id)) if tf.io.gfile.exists(model_dir): tf.gfile.DeleteRecursively(model_dir) if tf.io.gfile.exists(param_config_file): tf.gfile.Remove(param_config_file) conn.experiments( FLAGS.experiment_id).suggestions().delete(state="open")
def sigopt_generator(self, experiment): """Generate optimal parameter configurations using SigOpt.""" for _ in xrange(experiment.observation_budget): conn = Connection(client_token=self.client_token) suggestion = conn.experiments(experiment.id).suggestions().create() yield suggestion.assignments.to_json()
parser.add_argument('--observation_budget', type=int, default=20) parser.add_argument( '--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/user/profile") the_args = parser.parse_args() connection = Connection(client_token=the_args.client_token) # Create an experiment with one paramter, x experiment = connection.experiments().create( name="Basic Test experiment", parameters=[{ 'name': 'x', 'bounds': { 'max': 50.0, 'min': 0.0 }, 'type': 'double' }], observation_budget=the_args.observation_budget, ) print('Created experiment id {0}'.format(experiment.id)) # In a loop: receive a suggestion, evaluate the metric, report an observation for _ in range(experiment.observation_budget): suggestion = connection.experiments( experiment.id).suggestions().create() print('Evaluating at suggested assignments: {0}'.format( suggestion.assignments)) value = evaluate_metric(suggestion.assignments) print('Reporting observation of value: {0}'.format(value))
samples = 4 if args.smoke_test else 100 conn = Connection(client_token=os.environ["SIGOPT_KEY"]) experiment = conn.experiments().create( name="prior experiment example", parameters=[ { "name": "w1", "bounds": {"max": 1, "min": 0}, "prior": {"mean": 1 / 3, "name": "normal", "scale": 0.2}, "type": "double", }, { "name": "w2", "bounds": {"max": 1, "min": 0}, "prior": {"mean": 1 / 3, "name": "normal", "scale": 0.2}, "type": "double", }, ], metrics=[ dict(name="std", objective="minimize", strategy="optimize"), dict(name="average", strategy="store"), ], observation_budget=samples, parallel_bandwidth=1, ) algo = SigOptSearch( connection=conn, experiment_id=experiment.id,
# Get hyperparameters if with_architecture: exp_name = 'Multicriteria[failures] GPU-powered Sentiment Analysis (SGD + Architecture)' param_filepath = 'cnn_text/long_hyperparams.json' else: exp_name = 'Multicriteria[failures] GPU-powered Sentiment Analysis (SGD Only)' param_filepath='cnn_text/hyperparams.json' with open(param_filepath) as f: hyperparams = f.read() hyperparams = json.loads(hyperparams) experiment = conn.experiments().create( name=exp_name, parameters=hyperparams, observation_budget=30 * len(hyperparams), metrics=[{'name': 'accuracy'}, {'name': 'negative_train_time'}], ) print("Created experiment: https://sigopt.com/experiment/" + experiment.id) else: experiment = conn.experiments(experiment_id).fetch() # Optimization Loop data = get_data() def create_observation_dict(suggestion): start = time.time() accuracy = calculate_objective( suggestion.assignments, data,
(stdoutdata,stderrdata) = process.communicate() sys.stderr.write(stderrdata) return float(stdoutdata.strip()) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--command', required=True, help="The command to run the function whose parameters you would " "like to optimize. Should accept parameters as command line argument and output only the evaluated metric at the " "suggested point.") parser.add_argument('--experiment_id', required=True, help="The parameters of this experiment should be the " "same type and name of the command line arguments to your executable file.") parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens") the_args = parser.parse_args() connection = Connection(client_token=the_args.client_token) experiment = connection.experiments(the_args.experiment_id).fetch() connection.experiments(the_args.experiment_id).suggestions().delete(state="open") evaluator = SubProcessEvaluator(the_args.command) # In a loop: receive a suggestion, evaluate the metric, report an observation while True: suggestion = connection.experiments(experiment.id).suggestions().create() print('Evaluating at suggested assignments: {0}'.format(suggestion.assignments)) value = evaluator.evaluate_metric(suggestion.assignments) print('Reporting observation of value: {0}'.format(value)) connection.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value, )
# Learn more about authenticating the SigOpt API: # https://sigopt.com/docs/overview/authentication conn = Connection(client_token=SIGOPT_API_TOKEN) # Load dataset # We are using the iris dataset as an example iris = datasets.load_iris() X = iris.data y = iris.target # Create a SigOpt experiment for the Random Forest parameters experiment = conn.experiments().create( name="Random Forest (Python)", parameters=[ dict(name="max_features", type="int", bounds=dict(min=1, max=len(iris)-1)), dict(name="n_estimators", type="int", bounds=dict(min=1, max=100)), dict(name="min_samples_leaf", type="int", bounds=dict(min=1, max=10)) ] ) print("Created experiment: https://sigopt.com/experiment/" + experiment.id) # Our object metric is the mean of cross validated accuracies # We use cross validation to prevent overfitting def evaluate_model(assignments, X, y): # evaluate cross folds for accuracy cv = cross_validation.ShuffleSplit( X.shape[0], n_iter=5, test_size=0.3, ) classifier = RandomForestClassifier(
sleep_seconds = 10 print('{0} - Sleeping for {1} seconds to simulate expensive computation...'.format(threading.current_thread(), sleep_seconds)) time.sleep(sleep_seconds) return sigopt.examples.franke_function(assignments['x1'], assignments['x2']) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--runner_count', type=int, default=2) parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens") the_args = parser.parse_args() client_token = the_args.client_token conn = Connection(client_token=client_token) experiment = conn.experiments().create( name="Parallel Test Franke Function", project="sigopt-examples", parameters=[ {'name': 'x1', 'bounds': {'max': 1.0, 'min': 0.0}, 'type': 'double'}, {'name': 'x2', 'bounds': {'max': 1.0, 'min': 0.0}, 'type': 'double'}, ], ) runners = [ExampleRunner(client_token, experiment.id) for _ in range(the_args.runner_count)] for runner in runners: runner.daemon = True runner.start() for runner in runners: runner.join()
def evaluate_metric(assignments): x = assignments['x'] raise NotImplementedError("Return a number, which represents your metric evaluated at x") if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--observation_budget', type=int, default=20) parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens") the_args = parser.parse_args() connection = Connection(client_token=the_args.client_token) # Create an experiment with one paramter, x experiment = connection.experiments().create( name="Basic Test experiment", project="sigopt-examples", parameters=[{'name': 'x', 'bounds': {'max': 50.0, 'min': 0.0}, 'type': 'double'}], observation_budget=the_args.observation_budget, ) print('Created experiment id {0}'.format(experiment.id)) # In a loop: receive a suggestion, evaluate the metric, report an observation for _ in range(experiment.observation_budget): suggestion = connection.experiments(experiment.id).suggestions().create() print('Evaluating at suggested assignments: {0}'.format(suggestion.assignments)) value = evaluate_metric(suggestion.assignments) print('Reporting observation of value: {0}'.format(value)) connection.experiments(experiment.id).observations().create( suggestion=suggestion.id, value=value, )