def evaluate_metric(assignments): return evaluate(INPUT_FOLDERS_PXL, GT_FOLDERS_XML, GT_FOLDERS_PXL, OUTPUT_FOLDER, NUM_CORES, EVAL_TOOL, assignments['penalty_reduction'], assignments['seam_every_x_pxl'], assignments['small_component_ratio']) if __name__ == '__main__': # Real Token conn = Connection(client_token="") # Dev Token # conn = Connection(client_token="") # DEV!!!!!!!!!!!!! conn.set_api_url("https://api.sigopt.com") experiment = conn.experiments().create( name="HIP line segmentation v2", parameters=[ dict(name="penalty_reduction", type="int", bounds=dict(min=3000, max=13000)), dict(name="seam_every_x_pxl", type="int", bounds=dict(min=10, max=50)), dict(name="small_component_ratio", type="double", bounds=dict(min=0.1, max=0.5)), ], metrics=[dict(name="line IU")],
class SigOptTune(object): def __init__(self): """Constructor.""" self.logger = logging.getLogger(__name__) # get the logger! self.conn = Connection( client_token="XWCROUDALHMNJFABTLYVXBUHISZQKKACUGULCENHPSZNQPSD") self.conn.set_api_url("https://api.sigopt.com") self.experiment = None self.suggestion = None self.model = None self.acc = None def create_cnn_experiment(self): """Create experiment. Modify as needed.""" self.experiment = self.conn.experiments().create( name="CNNModel Accuracy v3", parameters=[ dict(name="learning_rate", bounds=dict(min=0.00001, max=0.1), type="double"), dict(name="dropout_rate", bounds=dict(min=0.2, max=0.9), type="double"), dict(name="beta1", bounds=dict(min=0.0001, max=0.999), type="double"), dict(name="beta2", bounds=dict(min=0.0001, max=0.999), type="double"), dict(name="epsilon", bounds=dict(min=1e-8, max=1.0), type="double"), dict(name="num_filt_1", bounds=dict(min=1, max=40), type="int"), dict(name="kernel_size", bounds=dict(min=1, max=10), type="int"), dict(name="num_fc_1", bounds=dict(min=1, max=40), type="int") ]) self.logger.info('Experiment created! ID %d.', self.experiment.id) def create_rnn_experiment(self): """Create experiment. Modify as needed.""" self.experiment = self.conn.experiments().create( name="RNNModel Accuracy v1", parameters=[ dict(name="learning_rate", bounds=dict(min=0.00001, max=0.1), type="double"), dict(name="dropout_rate", bounds=dict(min=0.2, max=0.9), type="double"), dict(name="beta1", bounds=dict(min=0.0001, max=0.999), type="double"), dict(name="beta2", bounds=dict(min=0.0001, max=0.999), type="double"), dict(name="epsilon", bounds=dict(min=1e-8, max=1.0), type="double"), dict(name="n_hidden", bounds=dict(min=1, max=40), type="int"), dict(name="num_fc_1", bounds=dict(min=1, max=40), type="int"), dict(name="n_layers", bounds=dict(min=1, max=10), type="int") ]) self.logger.info('Experiment created! ID %d.', self.experiment.id) def get_suggestions(self): """Create suggestions for next iteration.""" try: self.suggestion = self.conn.experiments( EXPERIMENT_ID).suggestions().create() logger.info('Created new suggestions.') except: self.conn.experiments(EXPERIMENT_ID).suggestions().delete( state="open") self.suggestion = self.conn.experiments( EXPERIMENT_ID).suggestions().create() logger.info('Deleted old and created new suggestions.') def update_parameters(self): """Update model parameters with suggestions.""" #model_type = self.model.__class__.__name__.replace('Model', '') params = self.suggestion.assignments # if model_type == 'CNN': # self.model.num_filt_1 = int(params['num_filt_1']) # self.model.kernel_size = int(params['kernel_size']) # self.model.num_fc_1 = int(params['num_fc_1']) # elif model_type == 'RNN': # self.model.n_hidden = int(params['n_hidden']) # self.model.num_fc_1 = int(params['num_fc_1']) # self.model.n_layers = int(params['n_layers']) #self.model.dropout_rate = params['dropout_rate'] self.model.learning_rate = params['learning_rate'] self.model.beta1 = params['beta1'] self.model.beta2 = params['beta2'] self.model.epsilon = params['epsilon'] def report_observation(self): """Report observation to SigOpt.""" self.conn.experiments(EXPERIMENT_ID).observations().create( suggestion=self.suggestion.id, value=float(self.acc), value_stddev=0.05) def optimization_loop(self, model): """Optimize the parameters based on suggestions.""" for i in range(100): self.logger.info('Optimization Loop Count: %d', i) # assign suggestions to parameters and hyperparameters self.get_suggestions() # update model class self.model = model() self.update_parameters() self.model.build_model() # update training class train = TrainModel(self.model, n_epochs=200, batch_size=128) # run the training stuff self.acc = train.train_model() train.reset_model() # report to SigOpt self.report_observation()