def evaluate_metric(assignments):
return evaluate(INPUT_FOLDERS_PXL, GT_FOLDERS_XML, GT_FOLDERS_PXL,
OUTPUT_FOLDER, NUM_CORES, EVAL_TOOL,
assignments['penalty_reduction'],
assignments['seam_every_x_pxl'],
assignments['small_component_ratio'])
if __name__ == '__main__':
# Real Token
conn = Connection(client_token="")
# Dev Token
# conn = Connection(client_token="") # DEV!!!!!!!!!!!!!
conn.set_api_url("https://api.sigopt.com")
experiment = conn.experiments().create(
name="HIP line segmentation v2",
parameters=[
dict(name="penalty_reduction",
type="int",
bounds=dict(min=3000, max=13000)),
dict(name="seam_every_x_pxl",
type="int",
bounds=dict(min=10, max=50)),
dict(name="small_component_ratio",
type="double",
bounds=dict(min=0.1, max=0.5)),
],
metrics=[dict(name="line IU")],
class SigOptTune(object):
def __init__(self):
"""Constructor."""
self.logger = logging.getLogger(__name__) # get the logger!
self.conn = Connection(
client_token="XWCROUDALHMNJFABTLYVXBUHISZQKKACUGULCENHPSZNQPSD")
self.conn.set_api_url("https://api.sigopt.com")
self.experiment = None
self.suggestion = None
self.model = None
self.acc = None
def create_cnn_experiment(self):
"""Create experiment. Modify as needed."""
self.experiment = self.conn.experiments().create(
name="CNNModel Accuracy v3",
parameters=[
dict(name="learning_rate",
bounds=dict(min=0.00001, max=0.1),
type="double"),
dict(name="dropout_rate",
bounds=dict(min=0.2, max=0.9),
type="double"),
dict(name="beta1",
bounds=dict(min=0.0001, max=0.999),
type="double"),
dict(name="beta2",
bounds=dict(min=0.0001, max=0.999),
type="double"),
dict(name="epsilon",
bounds=dict(min=1e-8, max=1.0),
type="double"),
dict(name="num_filt_1", bounds=dict(min=1, max=40),
type="int"),
dict(name="kernel_size",
bounds=dict(min=1, max=10),
type="int"),
dict(name="num_fc_1", bounds=dict(min=1, max=40), type="int")
])
self.logger.info('Experiment created! ID %d.', self.experiment.id)
def create_rnn_experiment(self):
"""Create experiment. Modify as needed."""
self.experiment = self.conn.experiments().create(
name="RNNModel Accuracy v1",
parameters=[
dict(name="learning_rate",
bounds=dict(min=0.00001, max=0.1),
type="double"),
dict(name="dropout_rate",
bounds=dict(min=0.2, max=0.9),
type="double"),
dict(name="beta1",
bounds=dict(min=0.0001, max=0.999),
type="double"),
dict(name="beta2",
bounds=dict(min=0.0001, max=0.999),
type="double"),
dict(name="epsilon",
bounds=dict(min=1e-8, max=1.0),
type="double"),
dict(name="n_hidden", bounds=dict(min=1, max=40), type="int"),
dict(name="num_fc_1", bounds=dict(min=1, max=40), type="int"),
dict(name="n_layers", bounds=dict(min=1, max=10), type="int")
])
self.logger.info('Experiment created! ID %d.', self.experiment.id)
def get_suggestions(self):
"""Create suggestions for next iteration."""
try:
self.suggestion = self.conn.experiments(
EXPERIMENT_ID).suggestions().create()
logger.info('Created new suggestions.')
except:
self.conn.experiments(EXPERIMENT_ID).suggestions().delete(
state="open")
self.suggestion = self.conn.experiments(
EXPERIMENT_ID).suggestions().create()
logger.info('Deleted old and created new suggestions.')
def update_parameters(self):
"""Update model parameters with suggestions."""
#model_type = self.model.__class__.__name__.replace('Model', '')
params = self.suggestion.assignments
# if model_type == 'CNN':
# self.model.num_filt_1 = int(params['num_filt_1'])
# self.model.kernel_size = int(params['kernel_size'])
# self.model.num_fc_1 = int(params['num_fc_1'])
# elif model_type == 'RNN':
# self.model.n_hidden = int(params['n_hidden'])
# self.model.num_fc_1 = int(params['num_fc_1'])
# self.model.n_layers = int(params['n_layers'])
#self.model.dropout_rate = params['dropout_rate']
self.model.learning_rate = params['learning_rate']
self.model.beta1 = params['beta1']
self.model.beta2 = params['beta2']
self.model.epsilon = params['epsilon']
def report_observation(self):
"""Report observation to SigOpt."""
self.conn.experiments(EXPERIMENT_ID).observations().create(
suggestion=self.suggestion.id,
value=float(self.acc),
value_stddev=0.05)
def optimization_loop(self, model):
"""Optimize the parameters based on suggestions."""
for i in range(100):
self.logger.info('Optimization Loop Count: %d', i)
# assign suggestions to parameters and hyperparameters
self.get_suggestions()
# update model class
self.model = model()
self.update_parameters()
self.model.build_model()
# update training class
train = TrainModel(self.model, n_epochs=200, batch_size=128)
# run the training stuff
self.acc = train.train_model()
train.reset_model()
# report to SigOpt
self.report_observation()
