class TestExperimentDaemon(unittest.TestCase):
exp_db: ExperimentDatabase
redis_resource: RedisResource = RedisResource()
def setUp(self) -> None:
self.rds = self.init_rds()
self.exp_db = self.init_db()
pymq.init(RedisConfig(self.rds))
self.recorder_factory = lambda exp_id: ExperimentTelemetryRecorder(
self.rds, self.exp_db, exp_id)
self.exp_ctrl = ExperimentController(self.rds)
self.cctrl = RedisClusterController(self.rds)
self.exp_service = SimpleExperimentService(self.exp_db)
def tearDown(self) -> None:
pymq.shutdown()
self.redis_resource.tearDown()
self.db_resource.tearDown()
@patch('galileo.experiment.runner.run_experiment')
@timeout_decorator.timeout(30)
def test_integration(self, mocked_run_experiment):
self.cctrl.register_worker('host1') # create a worker
daemon = ExperimentDaemon(self.rds, self.recorder_factory,
self.exp_ctrl, self.exp_service)
def inject_experiment():
exp = Experiment('experiment_id', creator='unittest')
cfg = ExperimentConfiguration(
2, 1,
[WorkloadConfiguration('aservice', [3, 5], 2, 'constant')])
queue = pymq.queue(ExperimentController.queue_key)
queue.put(QueuedExperiment(exp, cfg))
try:
poll(
lambda: queue.qsize() == 0, timeout=2,
interval=0.1) # wait for the daemon to take the experiment
finally:
daemon.close()
threading.Thread(target=inject_experiment).start()
daemon.run()
# wait for the experiment to be created
poll(lambda: self.exp_service.exists('experiment_id'),
timeout=2,
interval=0.1)
# wait for the experiment to be finished
poll(lambda: self.exp_service.find('experiment_id').status ==
'FINISHED',
timeout=3,
interval=0.1)
# verify that the experiment parameters were set correctly
exp = self.exp_service.find('experiment_id')
self.assertIsNotNone(exp)
self.assertEqual('FINISHED', exp.status)
self.assertEqual('experiment_id', exp.id)
self.assertEqual('experiment_id', exp.name)
# verify that experiment daemon tried to run the experiment
self.assertTrue(mocked_run_experiment.called,
'expected ExperimentDaemon to use experiment runner')
def init_rds(self):
self.redis_resource.setUp()
return self.redis_resource.rds
def init_db(self):
self.db_resource = SqliteResource()
self.db_resource.setUp()
return self.db_resource.db
class TestExperimentDaemon(unittest.TestCase):
redis_resource: RedisResource = RedisResource()
ectl: ExperimentController
cctl: ClusterController
def setUp(self) -> None:
self.rds = self.init_rds()
pymq.init(RedisConfig(self.rds))
self.ectl = ExperimentController(self.rds)
self.cctl = RedisClusterController(self.rds)
self.rds.delete(RedisClusterController.worker_key,
ExperimentController.queue_key)
def tearDown(self) -> None:
self.rds.delete(RedisClusterController.worker_key,
ExperimentController.queue_key)
pymq.shutdown()
self.redis_resource.tearDown()
def test_queue(self):
self.cctl.register_worker('host1')
exp = Experiment(name='my-experiment', creator='unittest')
config = ExperimentConfiguration(
2, 1, [WorkloadConfiguration('aservice', [1, 2], 2, 'constant')])
self.ectl.queue(config, exp)
queue = pymq.queue(ExperimentController.queue_key)
self.assertEqual(1, queue.qsize())
self.assertEqual(1, self.ectl.experiment_queue.qsize())
queued_experiment: QueuedExperiment = queue.get()
self.assertEqual(config, queued_experiment.configuration)
self.assertEqual('my-experiment', queued_experiment.experiment.name)
self.assertEqual('unittest', queued_experiment.experiment.creator)
def test_cancel(self):
self.cctl.register_worker('host1')
exp_id = 'abcd'
exp = Experiment(id=exp_id, name='my-experiment', creator='unittest')
config = ExperimentConfiguration(2, 1, [])
self.ectl.queue(config, exp)
exp = Experiment(id='abcdef',
name='my-experiment-2',
creator='unittest')
config = ExperimentConfiguration(2, 1, [])
self.ectl.queue(config, exp)
self.assertEqual(2, self.ectl.experiment_queue.qsize())
cancelled = self.ectl.cancel(exp_id)
self.assertTrue(cancelled)
self.assertEqual(1, self.ectl.experiment_queue.qsize())
queued = self.ectl.experiment_queue.get()
self.assertEqual('abcdef', queued.experiment.id,
'expected to find experiment that was not cancelled')
def init_rds(self):
self.redis_resource.setUp()
return self.redis_resource.rds