def setUp(self) -> None:
self.rds = self.init_rds()
pymq.init(RedisConfig(self.rds))
self.ectl = ExperimentController(self.rds)
self.cctl = RedisClusterController(self.rds)
self.rds.delete(RedisClusterController.worker_key,
ExperimentController.queue_key)
def setUp(self) -> None:
self.rds = self.init_rds()
self.exp_db = self.init_db()
pymq.init(RedisConfig(self.rds))
self.recorder_factory = lambda exp_id: ExperimentTelemetryRecorder(
self.rds, self.exp_db, exp_id)
self.exp_ctrl = ExperimentController(self.rds)
self.cctrl = RedisClusterController(self.rds)
self.exp_service = SimpleExperimentService(self.exp_db)
class ExperimentController:
queue_key = 'galileo:experiments:queue'
def __init__(self, rds: redis.Redis = None, eventbus=None) -> None:
super().__init__()
self.rds = rds or redis.Redis(decode_responses=True)
self.eventbus = eventbus or pymq
self.cluster = RedisClusterController(self.rds, eventbus=self.eventbus)
self.experiment_queue = ExperimentQueue(self.eventbus.queue(self.queue_key))
def queue(self, config: ExperimentConfiguration, exp: Experiment = None):
"""
Queues an experiment for the experiment daemon to load.
:param config: experiment configuration
:param exp: the experiment metadata (optional, as all parameters could be generated)
:return:
"""
if not self.cluster.list_workers():
raise ValueError('No workers to execute the experiment on')
element = QueuedExperiment(exp, config)
logger.debug('queuing experiment data: %s', element)
self.experiment_queue.put(element)
def cancel(self, exp_id: str) -> bool:
return self.experiment_queue.remove(exp_id)
def run_experiment(self, exp: Experiment, cfg: ExperimentConfiguration):
exp.status = 'IN_PROGRESS'
exp.start = time.time()
self.exp_service.save(exp)
with managed_recorder(self.create_recorder, exp.id):
logger.info("starting experiment %s", exp.id)
runner.run_experiment(RedisClusterController(self.rds), cfg)
def init_context(db: ExperimentSQLDatabase, rds: redis.Redis):
context = AppContext()
context.rds = rds
context.cctrl = RedisClusterController(context.rds)
context.ectrl = ExperimentController(context.rds)
context.exp_db = db
context.exp_service = SimpleExperimentService(context.exp_db)
context.repository = unittest.mock.MagicMock('repository')
return context
def create_context() -> AppContext:
context = AppContext()
context.rds = redis.Redis(host=os.getenv('galileo_redis_host',
'localhost'),
password=os.getenv('galileo_redis_password',
None),
port=int(os.getenv('galileo_redis_port', 6379)),
decode_responses=True)
pymq.init(RedisConfig(context.rds))
context.ectrl = ExperimentController(context.rds)
context.cctrl = RedisClusterController(context.rds)
context.exp_db = create_experiment_database_from_env()
context.exp_service = SimpleExperimentService(context.exp_db)
context.repository = Repository(
os.getenv('galileo_apps_repo_dir', os.path.abspath('./apps-repo')))
return context
class TestExperimentDaemon(unittest.TestCase):
exp_db: ExperimentDatabase
redis_resource: RedisResource = RedisResource()
def setUp(self) -> None:
self.rds = self.init_rds()
self.exp_db = self.init_db()
pymq.init(RedisConfig(self.rds))
self.recorder_factory = lambda exp_id: ExperimentTelemetryRecorder(
self.rds, self.exp_db, exp_id)
self.exp_ctrl = ExperimentController(self.rds)
self.cctrl = RedisClusterController(self.rds)
self.exp_service = SimpleExperimentService(self.exp_db)
def tearDown(self) -> None:
pymq.shutdown()
self.redis_resource.tearDown()
self.db_resource.tearDown()
@patch('galileo.experiment.runner.run_experiment')
@timeout_decorator.timeout(30)
def test_integration(self, mocked_run_experiment):
self.cctrl.register_worker('host1') # create a worker
daemon = ExperimentDaemon(self.rds, self.recorder_factory,
self.exp_ctrl, self.exp_service)
def inject_experiment():
exp = Experiment('experiment_id', creator='unittest')
cfg = ExperimentConfiguration(
2, 1,
[WorkloadConfiguration('aservice', [3, 5], 2, 'constant')])
queue = pymq.queue(ExperimentController.queue_key)
queue.put(QueuedExperiment(exp, cfg))
try:
poll(
lambda: queue.qsize() == 0, timeout=2,
interval=0.1) # wait for the daemon to take the experiment
finally:
daemon.close()
threading.Thread(target=inject_experiment).start()
daemon.run()
# wait for the experiment to be created
poll(lambda: self.exp_service.exists('experiment_id'),
timeout=2,
interval=0.1)
# wait for the experiment to be finished
poll(lambda: self.exp_service.find('experiment_id').status ==
'FINISHED',
timeout=3,
interval=0.1)
# verify that the experiment parameters were set correctly
exp = self.exp_service.find('experiment_id')
self.assertIsNotNone(exp)
self.assertEqual('FINISHED', exp.status)
self.assertEqual('experiment_id', exp.id)
self.assertEqual('experiment_id', exp.name)
# verify that experiment daemon tried to run the experiment
self.assertTrue(mocked_run_experiment.called,
'expected ExperimentDaemon to use experiment runner')
def init_rds(self):
self.redis_resource.setUp()
return self.redis_resource.rds
def init_db(self):
self.db_resource = SqliteResource()
self.db_resource.setUp()
return self.db_resource.db
class TestExperimentDaemon(unittest.TestCase):
redis_resource: RedisResource = RedisResource()
ectl: ExperimentController
cctl: ClusterController
def setUp(self) -> None:
self.rds = self.init_rds()
pymq.init(RedisConfig(self.rds))
self.ectl = ExperimentController(self.rds)
self.cctl = RedisClusterController(self.rds)
self.rds.delete(RedisClusterController.worker_key,
ExperimentController.queue_key)
def tearDown(self) -> None:
self.rds.delete(RedisClusterController.worker_key,
ExperimentController.queue_key)
pymq.shutdown()
self.redis_resource.tearDown()
def test_queue(self):
self.cctl.register_worker('host1')
exp = Experiment(name='my-experiment', creator='unittest')
config = ExperimentConfiguration(
2, 1, [WorkloadConfiguration('aservice', [1, 2], 2, 'constant')])
self.ectl.queue(config, exp)
queue = pymq.queue(ExperimentController.queue_key)
self.assertEqual(1, queue.qsize())
self.assertEqual(1, self.ectl.experiment_queue.qsize())
queued_experiment: QueuedExperiment = queue.get()
self.assertEqual(config, queued_experiment.configuration)
self.assertEqual('my-experiment', queued_experiment.experiment.name)
self.assertEqual('unittest', queued_experiment.experiment.creator)
def test_cancel(self):
self.cctl.register_worker('host1')
exp_id = 'abcd'
exp = Experiment(id=exp_id, name='my-experiment', creator='unittest')
config = ExperimentConfiguration(2, 1, [])
self.ectl.queue(config, exp)
exp = Experiment(id='abcdef',
name='my-experiment-2',
creator='unittest')
config = ExperimentConfiguration(2, 1, [])
self.ectl.queue(config, exp)
self.assertEqual(2, self.ectl.experiment_queue.qsize())
cancelled = self.ectl.cancel(exp_id)
self.assertTrue(cancelled)
self.assertEqual(1, self.ectl.experiment_queue.qsize())
queued = self.ectl.experiment_queue.get()
self.assertEqual('abcdef', queued.experiment.id,
'expected to find experiment that was not cancelled')
def init_rds(self):
self.redis_resource.setUp()
return self.redis_resource.rds
def __init__(self, rds: redis.Redis = None, eventbus=None) -> None:
super().__init__()
self.rds = rds or redis.Redis(decode_responses=True)
self.eventbus = eventbus or pymq
self.cluster = RedisClusterController(self.rds, eventbus=self.eventbus)
self.experiment_queue = ExperimentQueue(self.eventbus.queue(self.queue_key))