def _test_tuner():
_reverse_io() # now we are sending to Tuner's incoming stream
send(
CommandType.UpdateSearchSpace,
"{\"learning_rate\": {\"_value\": [0.0001, 0.001, 0.002, 0.005, 0.01], \"_type\": \"choice\"}, \"optimizer\": {\"_value\": [\"Adam\", \"SGD\"], \"_type\": \"choice\"}}"
)
send(CommandType.RequestTrialJobs, '2')
send(
CommandType.ReportMetricData,
'{"parameter_id":0,"type":"PERIODICAL","value":10,"trial_job_id":"abc"}'
)
send(CommandType.ReportMetricData,
'{"parameter_id":1,"type":"FINAL","value":11,"trial_job_id":"abc"}')
send(CommandType.AddCustomizedTrialJob, '{"param":-1}')
send(CommandType.ReportMetricData,
'{"parameter_id":2,"type":"FINAL","value":22,"trial_job_id":"abc"}')
send(CommandType.RequestTrialJobs, '1')
send(CommandType.TrialEnd, '{"trial_job_id":"abc"}')
_restore_io()
tuner = NaiveMultiPhaseTuner()
dispatcher = MultiPhaseMsgDispatcher(tuner)
dispatcher.run()
_reverse_io() # now we are receiving from Tuner's outgoing stream
command, data = receive() # this one is customized
print(command, data)
def test_tuner(self):
_reverse_io() # now we are sending to Tuner's incoming stream
send(CommandType.RequestTrialJobs, '2')
send(CommandType.ReportMetricData, '{"parameter_id":0,"type":"PERIODICAL","value":10}')
send(CommandType.ReportMetricData, '{"parameter_id":1,"type":"FINAL","value":11}')
send(CommandType.UpdateSearchSpace, '{"name":"SS0"}')
send(CommandType.AddCustomizedTrialJob, '{"param":-1}')
send(CommandType.ReportMetricData, '{"parameter_id":2,"type":"FINAL","value":22}')
send(CommandType.RequestTrialJobs, '1')
send(CommandType.KillTrialJob, 'null')
_restore_io()
tuner = NaiveTuner()
try:
tuner.run()
except Exception as e:
self.assertIs(type(e), AssertionError)
self.assertEqual(e.args[0], 'Unsupported command: CommandType.KillTrialJob')
_reverse_io() # now we are receiving from Tuner's outgoing stream
self._assert_params(0, 2, [ ], None)
self._assert_params(1, 4, [ ], None)
command, data = receive() # this one is customized
data = json.loads(data)
self.assertIs(command, CommandType.NewTrialJob)
self.assertEqual(data, {
'parameter_id': 2,
'parameter_source': 'customized',
'parameters': { 'param': -1 }
})
self._assert_params(3, 6, [[1,4,11,False], [2,-1,22,True]], {'name':'SS0'})
self.assertEqual(len(_out_buf.read()), 0) # no more commands
def test_assessor(self):
_reverse_io()
send(
CommandType.ReportMetricData,
'{"trial_job_id":"A","type":"PERIODICAL","sequence":0,"value":2}')
send(
CommandType.ReportMetricData,
'{"trial_job_id":"B","type":"PERIODICAL","sequence":0,"value":2}')
send(
CommandType.ReportMetricData,
'{"trial_job_id":"A","type":"PERIODICAL","sequence":1,"value":3}')
send(CommandType.TrialEnd,
'{"trial_job_id":"A","event":"SYS_CANCELED"}')
send(CommandType.TrialEnd, '{"trial_job_id":"B","event":"SUCCEEDED"}')
send(CommandType.NewTrialJob, 'null')
_restore_io()
assessor = NaiveAssessor()
dispatcher = MsgDispatcher(None, assessor)
try:
dispatcher.run()
except Exception as e:
self.assertIs(type(e), AssertionError)
self.assertEqual(e.args[0],
'Unsupported command: CommandType.NewTrialJob')
self.assertEqual(_trials, ['A', 'B', 'A'])
self.assertEqual(_end_trials, [('A', False), ('B', True)])
_reverse_io()
command, data = receive()
self.assertIs(command, CommandType.KillTrialJob)
self.assertEqual(data, '"A"')
self.assertEqual(len(_out_buf.read()), 0)
def _assert_params(self, parameter_id, param, trial_results, search_space):
command, data = receive()
self.assertIs(command, CommandType.NewTrialJob)
data = json.loads(data)
self.assertEqual(data['parameter_id'], parameter_id)
self.assertEqual(data['parameter_source'], 'algorithm')
self.assertEqual(data['parameters']['param'], param)
self.assertEqual(data['parameters']['trial_results'], trial_results)
self.assertEqual(data['parameters']['search_space'], search_space)
def test_receive_zh(self):
_prepare_receive('IN000006世界'.encode('utf8'))
command, data = receive()
self.assertIs(command, CommandType.Initialize)
self.assertEqual(data, '世界')
def test_receive_en(self):
_prepare_receive(b'IN000005hello')
command, data = receive()
self.assertIs(command, CommandType.Initialize)
self.assertEqual(data, 'hello')
