def handle_trial_end(self, data):
'''
data: it has three keys: trial_job_id, event, hyper_params
trial_job_id: the id generated by training service
event: the job's state
hyper_params: the hyperparameters (a string) generated and returned by tuner
'''
hyper_params = json_tricks.loads(data['hyper_params'])
bracket_id, i, _ = hyper_params['parameter_id'].split('_')
hyper_configs = self.brackets[int(bracket_id)].inform_trial_end(int(i))
if hyper_configs is not None:
_logger.debug('bracket %s next round %s, hyper_configs: %s',
bracket_id, i, hyper_configs)
self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs
for _ in range(self.credit):
if not self.generated_hyper_configs:
break
params = self.generated_hyper_configs.pop()
ret = {
'parameter_id': params[0],
'parameter_source': 'algorithm',
'parameters': params[1]
}
send(CommandType.NewTrialJob, json_tricks.dumps(ret))
self.credit -= 1
return True
def handle_report_metric_data(self, data):
trial_job_id = data['trial_job_id']
if data['type'] == 'FINAL':
id_ = data['parameter_id']
if id_ in _customized_parameter_ids:
self.tuner.receive_customized_trial_result(
id_, _trial_params[id_], data['value'], trial_job_id)
else:
self.tuner.receive_trial_result(id_, _trial_params[id_],
data['value'], trial_job_id)
elif data['type'] == 'PERIODICAL':
if self.assessor is not None:
self._handle_intermediate_metric_data(data)
else:
pass
elif data['type'] == 'REQUEST_PARAMETER':
assert data['trial_job_id'] is not None
assert data['parameter_index'] is not None
param_id = _create_parameter_id()
param = self.tuner.generate_parameters(param_id, trial_job_id)
send(
CommandType.SendTrialJobParameter,
_pack_parameter(param_id,
param,
trial_job_id=data['trial_job_id'],
parameter_index=data['parameter_index']))
else:
raise ValueError('Data type not supported: {}'.format(
data['type']))
return True
def _request_one_trial_job(self):
"""get one trial job, i.e., one hyperparameter configuration."""
if not self.generated_hyper_configs:
if self.curr_s < 0:
self.curr_s = self.s_max
_logger.debug('create a new bracket, self.curr_s=%d', self.curr_s)
self.brackets[self.curr_s] = Bracket(self.curr_s, self.s_max,
self.eta, self.R,
self.optimize_mode)
next_n, next_r = self.brackets[self.curr_s].get_n_r()
_logger.debug('new bracket, next_n=%d, next_r=%d', next_n, next_r)
assert self.searchspace_json is not None and self.random_state is not None
generated_hyper_configs = self.brackets[
self.curr_s].get_hyperparameter_configurations(
next_n, next_r, self.searchspace_json, self.random_state)
self.generated_hyper_configs = generated_hyper_configs.copy()
self.curr_s -= 1
assert self.generated_hyper_configs
params = self.generated_hyper_configs.pop()
ret = {
'parameter_id': params[0],
'parameter_source': 'algorithm',
'parameters': params[1]
}
send(CommandType.NewTrialJob, json_tricks.dumps(ret))
def _handle_intermediate_metric_data(self, data):
if data['type'] != 'PERIODICAL':
return True
if self.assessor is None:
return True
trial_job_id = data['trial_job_id']
if trial_job_id in _ended_trials:
return True
history = _trial_history[trial_job_id]
history[data['sequence']] = data['value']
ordered_history = _sort_history(history)
if len(ordered_history
) < data['sequence']: # no user-visible update since last time
return True
try:
result = self.assessor.assess_trial(trial_job_id, ordered_history)
except Exception as e:
_logger.exception('Assessor error')
if isinstance(result, bool):
result = AssessResult.Good if result else AssessResult.Bad
elif not isinstance(result, AssessResult):
msg = 'Result of Assessor.assess_trial must be an object of AssessResult, not %s'
raise RuntimeError(msg % type(result))
if result is AssessResult.Bad:
_logger.debug('BAD, kill %s', trial_job_id)
send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id))
else:
_logger.debug('GOOD')
def handle_initialize(self, data):
"""Initialize Tuner, including creating Bayesian optimization-based parametric models
and search space formations
Parameters
----------
data: search space
search space of this experiment
Raises
------
ValueError
Error: Search space is None
"""
logger.info('start to handle_initialize')
# convert search space jason to ConfigSpace
self.handle_update_search_space(data)
# generate BOHB config_generator using Bayesian optimization
if self.search_space:
self.cg = CG_BOHB(configspace=self.search_space,
min_points_in_model=self.min_points_in_model,
top_n_percent=self.top_n_percent,
num_samples=self.num_samples,
random_fraction=self.random_fraction,
bandwidth_factor=self.bandwidth_factor,
min_bandwidth=self.min_bandwidth)
else:
raise ValueError('Error: Search space is None')
# generate first brackets
self.generate_new_bracket()
send(CommandType.Initialized, '')
def handle_add_customized_trial(self, data):
# data: parameters
id_ = _create_parameter_id()
_customized_parameter_ids.add(id_)
send(CommandType.NewTrialJob,
_pack_parameter(id_, data, customized=True))
return True
def _request_one_trial_job(self):
"""get one trial job, i.e., one hyperparameter configuration.
If this function is called, Command will be sent by BOHB:
a. If there is a parameter need to run, will return "NewTrialJob" with a dict:
{
'parameter_id': id of new hyperparameter
'parameter_source': 'algorithm'
'parameters': value of new hyperparameter
}
b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with
{
'parameter_id': '-1_0_0',
'parameter_source': 'algorithm',
'parameters': ''
}
"""
if not self.generated_hyper_configs:
ret = {
'parameter_id': '-1_0_0',
'parameter_source': 'algorithm',
'parameters': ''
}
send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret))
return
assert self.generated_hyper_configs
params = self.generated_hyper_configs.pop()
ret = {
'parameter_id': params[0],
'parameter_source': 'algorithm',
'parameters': params[1]
}
self.parameters[params[0]] = params[1]
send(CommandType.NewTrialJob, json_tricks.dumps(ret))
self.credit -= 1
def handle_initialize(self, data):
logger.info("Advisor initialized: {}".format(data))
self.handle_update_search_space(data)
self.parameters_count = 0
self.parameter_best_metric = defaultdict(float)
self.parameter_cooldown = defaultdict(int)
send(CommandType.Initialized, '')
def handle_initialize(self, data):
'''
data is search space
'''
self.handle_update_search_space(data)
send(CommandType.Initialized, '')
return True
def test_send_too_large(self):
_prepare_send()
exception = None
try:
send(CommandType.NewTrialJob, ' ' * 1000000)
except AssertionError as e:
exception = e
self.assertIsNotNone(exception)
def handle_initialize(self, data):
"""callback for initializing the advisor
Parameters
----------
data: dict
search space
"""
self.handle_update_search_space(data)
send(CommandType.Initialized, '')
def handle_request_trial_jobs(self, data):
# data: number or trial jobs
ids = [_create_parameter_id() for _ in range(data)]
params_list = self.tuner.generate_multiple_parameters(ids)
assert len(ids) == len(params_list)
for i, _ in enumerate(ids):
send(CommandType.NewTrialJob,
_pack_parameter(ids[i], params_list[i]))
return True
def handle_initialize(self, data):
"""data is search space
Parameters
----------
data: int
number of trial jobs
"""
self.handle_update_search_space(data)
send(CommandType.Initialized, '')
def handle_request_trial_jobs(self, data):
"""
Parameters
----------
data: int
number of trial jobs
"""
for _ in range(data):
ret = self._get_one_trial_job()
send(CommandType.NewTrialJob, json_tricks.dumps(ret))
def handle_report_metric_data(self, data):
"""
Parameters
----------
data:
it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'.
Raises
------
ValueError
Data type not supported
"""
if data['type'] == MetricType.REQUEST_PARAMETER:
assert multi_phase_enabled()
assert data['trial_job_id'] is not None
assert data['parameter_index'] is not None
assert data['trial_job_id'] in self.job_id_para_id_map
self._handle_trial_end(
self.job_id_para_id_map[data['trial_job_id']])
ret = self._get_one_trial_job()
if data['trial_job_id'] is not None:
ret['trial_job_id'] = data['trial_job_id']
if data['parameter_index'] is not None:
ret['parameter_index'] = data['parameter_index']
self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id']
send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret))
else:
value = extract_scalar_reward(data['value'])
bracket_id, i, _ = data['parameter_id'].split('_')
bracket_id = int(bracket_id)
# add <trial_job_id, parameter_id> to self.job_id_para_id_map here,
# because when the first parameter_id is created, trial_job_id is not known yet.
if data['trial_job_id'] in self.job_id_para_id_map:
assert self.job_id_para_id_map[
data['trial_job_id']] == data['parameter_id']
else:
self.job_id_para_id_map[
data['trial_job_id']] = data['parameter_id']
if data['type'] == MetricType.FINAL:
# sys.maxsize indicates this value is from FINAL metric data, because data['sequence'] from FINAL metric
# and PERIODICAL metric are independent, thus, not comparable.
self.brackets[bracket_id].set_config_perf(
int(i), data['parameter_id'], sys.maxsize, value)
self.completed_hyper_configs.append(data)
elif data['type'] == MetricType.PERIODICAL:
self.brackets[bracket_id].set_config_perf(
int(i), data['parameter_id'], data['sequence'], value)
else:
raise ValueError('Data type not supported: {}'.format(
data['type']))
def _send_new_trial(self):
while self.unsatisfied_jobs:
ret = self._get_one_trial_job()
if ret is None:
break
one_unsatisfied = self.unsatisfied_jobs.pop(0)
ret['trial_job_id'] = one_unsatisfied['trial_job_id']
ret['parameter_index'] = one_unsatisfied['parameter_index']
# update parameter_id in self.job_id_para_id_map
self.job_id_para_id_map[ret['trial_job_id']] = ret['parameter_id']
send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret))
for _ in range(self.credit):
self._request_one_trial_job()
def _send_new_trial(self):
self.parameters_count += 1
new_trial = {
"parameter_id": self.parameters_count,
"parameters": {
"optimizer": param.choice(self.searchspace_json["optimizer"], self.random_state),
"learning_rate": param.loguniform(self.searchspace_json["learning_rate"][0],
self.searchspace_json["learning_rate"][1],
self.random_state)
},
"parameter_source": "algorithm"
}
logger.info("New trial sent: {}".format(new_trial))
send(CommandType.NewTrialJob, json_tricks.dumps(new_trial))
def handle_report_metric_data(self, data):
logger.info("Metric reported: {}".format(data))
if data['type'] == MetricType.REQUEST_PARAMETER:
raise ValueError("Request parameter not supported")
elif data["type"] == MetricType.PERIODICAL:
parameter_id = data["parameter_id"]
if data["value"] > self.parameter_best_metric[parameter_id]:
self.parameter_best_metric[parameter_id] = data["value"]
self.parameter_cooldown[parameter_id] = 0
else:
self.parameter_cooldown[parameter_id] += 1
logger.info("Accuracy dropped, cooldown {}, sending a new trial".format(
self.parameter_cooldown[parameter_id]))
self._send_new_trial()
if self.parameter_cooldown[parameter_id] >= self.k:
logger.info("Send kill signal to {}".format(data))
send(CommandType.KillTrialJob, json_tricks.dumps(data["trial_job_id"]))
def _request_one_trial_job(self):
"""get one trial job, i.e., one hyperparameter configuration.
If this function is called, Command will be sent by BOHB:
a. If there is a parameter need to run, will return "NewTrialJob" with a dict:
{
'parameter_id': id of new hyperparameter
'parameter_source': 'algorithm'
'parameters': value of new hyperparameter
}
b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with
{
'parameter_id': '-1_0_0',
'parameter_source': 'algorithm',
'parameters': ''
}
"""
ret = self._get_one_trial_job()
if ret is not None:
send(CommandType.NewTrialJob, json_tricks.dumps(ret))
self.credit -= 1
def test_send_zh(self):
out_file = _prepare_send()
send(CommandType.NewTrialJob, '你好')
self.assertEqual(out_file.getvalue(), 'TR000006你好'.encode('utf8'))
def test_send_en(self):
out_file = _prepare_send()
send(CommandType.NewTrialJob, 'CONTENT')
self.assertEqual(out_file.getvalue(), b'TR000007CONTENT')
def test_assessor(self):
_reverse_io()
send(
CommandType.ReportMetricData,
'{"trial_job_id":"A","type":"PERIODICAL","sequence":0,"value":2}')
send(
CommandType.ReportMetricData,
'{"trial_job_id":"B","type":"PERIODICAL","sequence":0,"value":2}')
send(
CommandType.ReportMetricData,
'{"trial_job_id":"A","type":"PERIODICAL","sequence":1,"value":3}')
send(CommandType.TrialEnd,
'{"trial_job_id":"A","event":"SYS_CANCELED"}')
send(CommandType.TrialEnd, '{"trial_job_id":"B","event":"SUCCEEDED"}')
send(CommandType.NewTrialJob, 'null')
_restore_io()
assessor = NaiveAssessor()
dispatcher = MsgDispatcher(None, assessor)
try:
dispatcher.run()
except Exception as e:
self.assertIs(type(e), AssertionError)
self.assertEqual(e.args[0],
'Unsupported command: CommandType.NewTrialJob')
self.assertEqual(_trials, ['A', 'B', 'A'])
self.assertEqual(_end_trials, [('A', False), ('B', True)])
_reverse_io()
command, data = receive()
self.assertIs(command, CommandType.KillTrialJob)
self.assertEqual(data, '"A"')
self.assertEqual(len(_out_buf.read()), 0)
def _test_tuner():
_reverse_io() # now we are sending to Tuner's incoming stream
send(
CommandType.UpdateSearchSpace,
"{\"learning_rate\": {\"_value\": [0.0001, 0.001, 0.002, 0.005, 0.01], \"_type\": \"choice\"}, \"optimizer\": {\"_value\": [\"Adam\", \"SGD\"], \"_type\": \"choice\"}}"
)
send(CommandType.RequestTrialJobs, '2')
send(
CommandType.ReportMetricData,
'{"parameter_id":0,"type":"PERIODICAL","value":10,"trial_job_id":"abc"}'
)
send(CommandType.ReportMetricData,
'{"parameter_id":1,"type":"FINAL","value":11,"trial_job_id":"abc"}')
send(CommandType.AddCustomizedTrialJob, '{"param":-1}')
send(CommandType.ReportMetricData,
'{"parameter_id":2,"type":"FINAL","value":22,"trial_job_id":"abc"}')
send(CommandType.RequestTrialJobs, '1')
send(CommandType.TrialEnd, '{"trial_job_id":"abc"}')
_restore_io()
tuner = NaiveMultiPhaseTuner()
dispatcher = MultiPhaseMsgDispatcher(tuner)
dispatcher.run()
_reverse_io() # now we are receiving from Tuner's outgoing stream
command, data = receive() # this one is customized
print(command, data)
def handle_report_metric_data(self, data):
"""reveice the metric data and update Bayesian optimization with final result
Parameters
----------
data:
it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'.
Raises
------
ValueError
Data type not supported
"""
logger.debug('handle report metric data = %s', data)
if data['type'] == MetricType.REQUEST_PARAMETER:
assert multi_phase_enabled()
assert data['trial_job_id'] is not None
assert data['parameter_index'] is not None
assert data['trial_job_id'] in self.job_id_para_id_map
self._handle_trial_end(
self.job_id_para_id_map[data['trial_job_id']])
ret = self._get_one_trial_job()
if ret is None:
self.unsatisfied_jobs.append({
'trial_job_id':
data['trial_job_id'],
'parameter_index':
data['parameter_index']
})
else:
ret['trial_job_id'] = data['trial_job_id']
ret['parameter_index'] = data['parameter_index']
# update parameter_id in self.job_id_para_id_map
self.job_id_para_id_map[
data['trial_job_id']] = ret['parameter_id']
send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret))
else:
assert 'value' in data
value = extract_scalar_reward(data['value'])
if self.optimize_mode is OptimizeMode.Maximize:
reward = -value
else:
reward = value
assert 'parameter_id' in data
s, i, _ = data['parameter_id'].split('_')
logger.debug('bracket id = %s, metrics value = %s, type = %s', s,
value, data['type'])
s = int(s)
# add <trial_job_id, parameter_id> to self.job_id_para_id_map here,
# because when the first parameter_id is created, trial_job_id is not known yet.
if data['trial_job_id'] in self.job_id_para_id_map:
assert self.job_id_para_id_map[
data['trial_job_id']] == data['parameter_id']
else:
self.job_id_para_id_map[
data['trial_job_id']] = data['parameter_id']
assert 'type' in data
if data['type'] == MetricType.FINAL:
# and PERIODICAL metric are independent, thus, not comparable.
assert 'sequence' in data
self.brackets[s].set_config_perf(int(i), data['parameter_id'],
sys.maxsize, value)
self.completed_hyper_configs.append(data)
_parameters = self.parameters[data['parameter_id']]
_parameters.pop(_KEY)
# update BO with loss, max_s budget, hyperparameters
self.cg.new_result(loss=reward,
budget=data['sequence'],
parameters=_parameters,
update_model=True)
elif data['type'] == MetricType.PERIODICAL:
self.brackets[s].set_config_perf(int(i), data['parameter_id'],
data['sequence'], value)
else:
raise ValueError('Data type not supported: {}'.format(
data['type']))
def test_tuner(self):
_reverse_io() # now we are sending to Tuner's incoming stream
send(CommandType.RequestTrialJobs, '2')
send(CommandType.ReportMetricData, '{"parameter_id":0,"type":"PERIODICAL","value":10}')
send(CommandType.ReportMetricData, '{"parameter_id":1,"type":"FINAL","value":11}')
send(CommandType.UpdateSearchSpace, '{"name":"SS0"}')
send(CommandType.AddCustomizedTrialJob, '{"param":-1}')
send(CommandType.ReportMetricData, '{"parameter_id":2,"type":"FINAL","value":22}')
send(CommandType.RequestTrialJobs, '1')
send(CommandType.KillTrialJob, 'null')
_restore_io()
tuner = NaiveTuner()
try:
tuner.run()
except Exception as e:
self.assertIs(type(e), AssertionError)
self.assertEqual(e.args[0], 'Unsupported command: CommandType.KillTrialJob')
_reverse_io() # now we are receiving from Tuner's outgoing stream
self._assert_params(0, 2, [ ], None)
self._assert_params(1, 4, [ ], None)
command, data = receive() # this one is customized
data = json.loads(data)
self.assertIs(command, CommandType.NewTrialJob)
self.assertEqual(data, {
'parameter_id': 2,
'parameter_source': 'customized',
'parameters': { 'param': -1 }
})
self._assert_params(3, 6, [[1,4,11,False], [2,-1,22,True]], {'name':'SS0'})
self.assertEqual(len(_out_buf.read()), 0) # no more commands
def test_msg_dispatcher(self):
_reverse_io() # now we are sending to Tuner's incoming stream
send(CommandType.RequestTrialJobs, '2')
send(CommandType.ReportMetricData,
'{"parameter_id":0,"type":"PERIODICAL","value":10}')
send(CommandType.ReportMetricData,
'{"parameter_id":1,"type":"FINAL","value":11}')
send(CommandType.UpdateSearchSpace, '{"name":"SS0"}')
send(CommandType.RequestTrialJobs, '1')
send(CommandType.KillTrialJob, 'null')
_restore_io()
tuner = NaiveTuner()
dispatcher = MsgDispatcher(tuner)
nni.msg_dispatcher_base._worker_fast_exit_on_terminate = False
dispatcher.run()
e = dispatcher.worker_exceptions[0]
self.assertIs(type(e), AssertionError)
self.assertEqual(e.args[0],
'Unsupported command: CommandType.KillTrialJob')
_reverse_io() # now we are receiving from Tuner's outgoing stream
self._assert_params(0, 2, [], None)
self._assert_params(1, 4, [], None)
self._assert_params(2, 6, [[1, 4, 11, False]], {'name': 'SS0'})
self.assertEqual(len(_out_buf.read()), 0) # no more commands
