class TestTraceDatabaseLogger(AbstractTraceLoggerTestCase, unittest.TestCase):
sql_resource = SqliteResource()
def setUp(self) -> None:
self.sql_resource.setUp()
self.queue = multiprocessing.Queue()
self.logger = TraceDatabaseLogger(self.queue, self.sql_resource.db)
self.logger.flush_interval = 2
self.flush_interval = 2
self.thread = threading.Thread(target=self.logger.run)
self.thread.start()
def tearDown(self) -> None:
if not self.logger.closed:
self.logger.close()
self.thread.join(2)
self.sql_resource.tearDown()
@timeout_decorator.timeout(5)
def assert_flush(self, n):
traces = self.sql_resource.sql.fetchall('SELECT * FROM `traces`')
self.assertEqual(len(traces), n)
@timeout_decorator.timeout(5)
def count_traces(self) -> int:
return len(self.sql_resource.sql.fetchall('SELECT * FROM `traces`'))
class TestExperimentTelemetryRecorder(unittest.TestCase):
redis_resource: RedisResource = RedisResource()
db_resource: SqliteResource = SqliteResource()
exp_db: ExperimentSQLDatabase
reporter: RedisTelemetryReporter
def setUp(self) -> None:
self.redis_resource.setUp()
self.db_resource.setUp()
self.reporter = RedisTelemetryReporter(self.redis_resource.rds)
def tearDown(self) -> None:
self.redis_resource.tearDown()
self.db_resource.tearDown()
@timeout_decorator.timeout(5)
def test_recorder_records_correctly(self):
recorder = ExperimentTelemetryRecorder(self.redis_resource.rds,
self.db_resource.db,
'unittest',
flush_every=1)
recorder._record(Telemetry('1.0', '31', 'node1', 'cpu'))
recorder._record(Telemetry('2.0', '32', 'node2', 'cpu'))
recorder._record(Telemetry('3.0', '33', 'node2', 'rx', 'eth0'))
records = self.db_resource.sql.fetchall(
'SELECT * FROM `telemetry` WHERE EXP_ID = "unittest"')
self.assertEqual(3, len(records))
self.assertEqual(('unittest', 1.0, 'cpu', None, 'node1', 31.0),
records[0])
self.assertEqual(('unittest', 2.0, 'cpu', None, 'node2', 32.0),
records[1])
self.assertEqual(('unittest', 3.0, 'rx', 'eth0', 'node2', 33.0),
records[2])
@timeout_decorator.timeout(5)
def test_publish_non_float_value_does_not_break_recorder(self):
recorder = ExperimentTelemetryRecorder(self.redis_resource.rds,
self.db_resource.db, 'unittest')
recorder.start()
time.sleep(0.5)
try:
self.reporter.report(Telemetry('5', '35', 'node1', 'cpu'))
self.reporter.report(Telemetry('6', 'foo', 'node1', 'cpu'))
self.reporter.report(Telemetry('7', '37', 'node2', 'cpu'))
finally:
time.sleep(0.5)
recorder.stop(timeout=2)
records = self.db_resource.sql.fetchall(
'SELECT * FROM `telemetry` WHERE EXP_ID = "unittest"')
self.assertEqual(2, len(records))
self.assertEqual(('unittest', 5.0, 'cpu', None, 'node1', 35.0),
records[0])
self.assertEqual(('unittest', 7.0, 'cpu', None, 'node2', 37.0),
records[1])
class ResourceTest(testing.TestCase):
redis_resource: RedisResource = RedisResource()
ctx: AppContext
def setUp(self):
super(ResourceTest, self).setUp()
rds = self.init_rds()
db = self.init_db()
self.ctx = self.init_context(db, rds)
self.app = self.create_api(self.ctx)
def tearDown(self) -> None:
super(ResourceTest, self).tearDown()
pymq.shutdown()
self.redis_resource.tearDown()
self.db_resource.tearDown()
def init_db(self):
self.db_resource = SqliteResource()
self.db_resource.setUp()
return self.db_resource.db
def init_rds(self):
self.redis_resource.setUp()
pymq.init(RedisConfig(self.redis_resource.rds))
return self.redis_resource.rds
def create_api(self, ctx) -> falcon.API:
api = falcon.API(middleware=[CORSComponent()])
setup(api, ctx)
return api
@staticmethod
def init_context(db: ExperimentSQLDatabase, rds: redis.Redis):
context = AppContext()
context.rds = rds
context.cctrl = RedisClusterController(context.rds)
context.ectrl = ExperimentController(context.rds)
context.exp_db = db
context.exp_service = SimpleExperimentService(context.exp_db)
context.repository = unittest.mock.MagicMock('repository')
return context
class TestDatabaseTraceWriter(unittest.TestCase):
sql_resource = SqliteResource()
def setUp(self) -> None:
self.sql_resource.setUp()
def tearDown(self) -> None:
self.sql_resource.tearDown()
@timeout_decorator.timeout(5)
def test_write(self):
self.writer = DatabaseTraceWriter(self.sql_resource.db)
self.writer.write(traces)
actual = self.sql_resource.db.get_traces()
self.assertEqual(3, len(actual))
self.assertEqual(traces[0], actual[0])
self.assertEqual(traces[1], actual[1])
self.assertEqual(traces[2], actual[2])
def init_db(self):
self.db_resource = SqliteResource()
self.db_resource.setUp()
return self.db_resource.db
class TestExperimentDaemon(unittest.TestCase):
exp_db: ExperimentDatabase
redis_resource: RedisResource = RedisResource()
def setUp(self) -> None:
self.rds = self.init_rds()
self.exp_db = self.init_db()
pymq.init(RedisConfig(self.rds))
self.recorder_factory = lambda exp_id: ExperimentTelemetryRecorder(
self.rds, self.exp_db, exp_id)
self.exp_ctrl = ExperimentController(self.rds)
self.cctrl = RedisClusterController(self.rds)
self.exp_service = SimpleExperimentService(self.exp_db)
def tearDown(self) -> None:
pymq.shutdown()
self.redis_resource.tearDown()
self.db_resource.tearDown()
@patch('galileo.experiment.runner.run_experiment')
@timeout_decorator.timeout(30)
def test_integration(self, mocked_run_experiment):
self.cctrl.register_worker('host1') # create a worker
daemon = ExperimentDaemon(self.rds, self.recorder_factory,
self.exp_ctrl, self.exp_service)
def inject_experiment():
exp = Experiment('experiment_id', creator='unittest')
cfg = ExperimentConfiguration(
2, 1,
[WorkloadConfiguration('aservice', [3, 5], 2, 'constant')])
queue = pymq.queue(ExperimentController.queue_key)
queue.put(QueuedExperiment(exp, cfg))
try:
poll(
lambda: queue.qsize() == 0, timeout=2,
interval=0.1) # wait for the daemon to take the experiment
finally:
daemon.close()
threading.Thread(target=inject_experiment).start()
daemon.run()
# wait for the experiment to be created
poll(lambda: self.exp_service.exists('experiment_id'),
timeout=2,
interval=0.1)
# wait for the experiment to be finished
poll(lambda: self.exp_service.find('experiment_id').status ==
'FINISHED',
timeout=3,
interval=0.1)
# verify that the experiment parameters were set correctly
exp = self.exp_service.find('experiment_id')
self.assertIsNotNone(exp)
self.assertEqual('FINISHED', exp.status)
self.assertEqual('experiment_id', exp.id)
self.assertEqual('experiment_id', exp.name)
# verify that experiment daemon tried to run the experiment
self.assertTrue(mocked_run_experiment.called,
'expected ExperimentDaemon to use experiment runner')
def init_rds(self):
self.redis_resource.setUp()
return self.redis_resource.rds
def init_db(self):
self.db_resource = SqliteResource()
self.db_resource.setUp()
return self.db_resource.db
class TestExperimentEventRecorder(unittest.TestCase):
redis_resource: RedisResource = RedisResource()
db_resource: SqliteResource = SqliteResource()
exp_db: ExperimentSQLDatabase
reporter: RedisEventReporter
def setUp(self) -> None:
self.redis_resource.setUp()
self.db_resource.setUp()
self.reporter = RedisEventReporter(self.redis_resource.rds)
def tearDown(self) -> None:
self.redis_resource.tearDown()
self.db_resource.tearDown()
def test_batching_recorder_records_correctly(self):
thread = ExperimentEventRecorderThread(
BatchingExperimentEventRecorder(self.redis_resource.rds, self.db_resource.db, 'unittest', flush_every=1)
)
thread.start()
time.sleep(0.1)
thread.recorder._record(Event(1., 'start', 'function1'))
thread.recorder._record(Event(2., 'stop', 'function1'))
thread.recorder._record(Event(3., 'exit'))
thread.stop()
records = self.db_resource.sql.fetchall('SELECT * FROM `events` WHERE EXP_ID = "unittest"')
self.assertEqual(3, len(records))
self.assertEqual(ExperimentEvent('unittest', 1., 'start', 'function1'), records[0])
self.assertEqual(ExperimentEvent('unittest', 2., 'stop', 'function1'), records[1])
self.assertEqual(ExperimentEvent('unittest', 3., 'exit'), records[2])
def test_batching_recorder_flush_after_stop(self):
thread = ExperimentEventRecorderThread(
BatchingExperimentEventRecorder(self.redis_resource.rds, self.db_resource.db, 'unittest', flush_every=5)
)
thread.start()
time.sleep(0.1)
thread.recorder._record(Event(1., 'start', 'function1'))
thread.recorder._record(Event(2., 'stop', 'function1'))
thread.recorder._record(Event(3., 'exit'))
thread.stop()
records = self.db_resource.sql.fetchall('SELECT * FROM `events` WHERE EXP_ID = "unittest"')
self.assertEqual(3, len(records))
self.assertEqual(ExperimentEvent('unittest', 1., 'start', 'function1'), records[0])
self.assertEqual(ExperimentEvent('unittest', 2., 'stop', 'function1'), records[1])
self.assertEqual(ExperimentEvent('unittest', 3., 'exit'), records[2])
def test_batching_recorder_with_redis(self):
thread = ExperimentEventRecorderThread(
BatchingExperimentEventRecorder(self.redis_resource.rds, self.db_resource.db, 'unittest', flush_every=1)
)
thread.start()
time.sleep(0.5)
self.redis_resource.rds.publish("galileo/events", "1. start function1")
self.redis_resource.rds.publish("galileo/events", "2. stop function1")
self.redis_resource.rds.publish("galileo/events", "3. exit")
time.sleep(0.5)
thread.stop()
records = self.db_resource.sql.fetchall('SELECT * FROM `events` WHERE EXP_ID = "unittest"')
self.assertEqual(3, len(records))
self.assertEqual(ExperimentEvent('unittest', 1., 'start', 'function1'), records[0])
self.assertEqual(ExperimentEvent('unittest', 2., 'stop', 'function1'), records[1])
self.assertEqual(ExperimentEvent('unittest', 3., 'exit'), records[2])
def test_recorder_with_redis(self):
thread = ExperimentEventRecorderThread(
ExperimentEventRecorder(self.redis_resource.rds, self.db_resource.db, 'unittest')
)
thread.start()
time.sleep(0.5)
self.redis_resource.rds.publish("galileo/events", "1. start function1")
self.redis_resource.rds.publish("galileo/events", "2. stop function1")
self.redis_resource.rds.publish("galileo/events", "3. exit")
time.sleep(0.5)
thread.stop()
records = self.db_resource.sql.fetchall('SELECT * FROM `events` WHERE EXP_ID = "unittest"')
self.assertEqual(3, len(records))
self.assertEqual(ExperimentEvent('unittest', 1., 'start', 'function1'), records[0])
self.assertEqual(ExperimentEvent('unittest', 2., 'stop', 'function1'), records[1])
self.assertEqual(ExperimentEvent('unittest', 3., 'exit'), records[2])