def test_delete_experiment_removes_telemetry(self):
exp_id = 'expid10'
exp_id_control = 'expid11'
self.db.save_experiment(
Experiment(exp_id, 'test_experiment', 'unittest', 10, 100, 1,
'finished'))
self.db.save_telemetry([Telemetry(1, 'cpu', 'n1', 32, exp_id)])
self.db.save_experiment(
Experiment(exp_id_control, 'test_experiment', 'unittest', 10, 100,
1, 'finished'))
self.db.save_telemetry([Telemetry(1, 'cpu', 'n1', 32, exp_id_control)])
self.assertIsNotNone(self.db.get_experiment(exp_id))
telemetry_rows = self.db.db.fetchall(
'SELECT * FROM telemetry WHERE EXP_ID = "%s"' % exp_id)
self.assertEqual(1, len(telemetry_rows))
self.db.delete_experiment(exp_id)
self.assertIsNone(self.db.get_experiment(exp_id))
telemetry_rows = self.db.db.fetchall(
'SELECT * FROM telemetry WHERE EXP_ID = "%s"' % exp_id)
self.assertEqual(0, len(telemetry_rows))
self.assertIsNotNone(self.db.get_experiment(exp_id_control))
telemetry_rows = self.db.db.fetchall(
'SELECT * FROM telemetry WHERE EXP_ID = "%s"' % exp_id_control)
self.assertEqual(1, len(telemetry_rows))
def on_post(self, req: falcon.Request, resp):
if not self.cctrl.list_workers():
raise falcon.HTTPServiceUnavailable(
'no available hosts to execute the experiment')
doc = req.media
exp = doc['experiment'] if 'experiment' in doc else dict()
if 'id' not in exp:
exp['id'] = generate_experiment_id()
exp = Experiment(**exp)
exp.created = time.time()
exp.status = 'QUEUED'
self.exp_service.save(exp)
logger.debug('deserialized experiment %s', exp)
workloads = [
WorkloadConfiguration(**workload)
for workload in doc['configuration']['workloads']
]
duration = to_seconds(doc['configuration']['duration'])
interval = to_seconds(doc['configuration']['interval'])
config = ExperimentConfiguration(duration, interval, workloads)
logger.debug('deserialized experiment config %s', config)
logger.debug('queuing experiment with id %s', exp.id)
self.ectrl.queue(config, exp)
resp.media = exp.id
def run_experiment(self, exp: Experiment, cfg: ExperimentConfiguration):
exp.status = 'IN_PROGRESS'
exp.start = time.time()
self.exp_service.save(exp)
with managed_recorder(self.create_recorder, exp.id):
logger.info("starting experiment %s", exp.id)
runner.run_experiment(RedisClusterController(self.rds), cfg)
def test_update_experiment_and_get(self):
exp = Experiment('expid8', 'test_experiment', 'unittest', 10, None, 1,
'running')
self.db.save_experiment(exp)
self.assertEqual(self.db.get_experiment('expid8').status, 'running')
self.assertEqual(self.db.get_experiment('expid8').end, None)
exp.status = 'finished'
exp.end = 100
self.db.update_experiment(exp)
self.assertEqual(self.db.get_experiment('expid8').status, 'finished')
self.assertEqual(self.db.get_experiment('expid8').end, 100)
def test_get(self):
exp = Experiment('id1', 'name1', 'creator1', 123, 456, 789, 'FINISH')
self.db_resource.db.save_experiment(exp)
result = self.simulate_get(f'/api/experiments/id1')
self.assertIsNotNone(result.json)
self.assertEqual(result.json, exp.__dict__)
def test_delete(self):
exp_id = 'id1'
exp = Experiment(exp_id, 'name1', 'creator1', 1, 20, 7, 'FINISH')
telemetry: List[Telemetry] = [
Telemetry(1, 'metric1', 'node1', 1, exp_id),
Telemetry(2, 'metric2', 'node2', 2, exp_id)
]
traces: List[RequestTrace] = [
RequestTrace('req1', 'client1', 'service1', 2, 2, 3),
RequestTrace('req2', 'client2', 'service2', 6, 5, 4)
]
db = self.db_resource.db
db.save_experiment(exp)
db.save_telemetry(telemetry)
db.save_traces(traces)
db.touch_traces(exp)
self.simulate_delete('/api/experiments/id1')
self.assertEqual(len(db.find_all()), 0)
traces_fetched = db.db.fetchall('SELECT * FROM traces')
telemetry_fetched = db.db.fetchall('SELECT * FROM telemetry')
self.assertEqual(len(traces_fetched), 0)
self.assertEqual(len(telemetry_fetched), 0)
pass
def get_experiment(self, exp_id: str) -> Experiment:
sql = "SELECT * FROM `experiments` WHERE EXP_ID = " + self.db.placeholder
entry = self.db.fetchone(sql, (exp_id, ))
if entry:
row = tuple(entry)
return Experiment(*row)
else:
return None
def find_all(self) -> List[Experiment]:
fields = [
'exp_id', 'name', 'creator', 'start', 'end', 'created', 'status'
]
fields = self.db.sql_field_list(fields)
sql = f'SELECT {fields} FROM `experiments`'
entries = self.db.fetchall(sql)
return list(map(lambda x: Experiment(*(tuple(x))), entries))
def test_update_and_get(self):
self.db.save_experiment(
Experiment('expid6', 'test_experiment', 'unittest', 10, 100, 1,
'finished'))
self.db.save_experiment(
Experiment('expid7', 'test_experiment', 'unittest', 100, None, 10,
'running'))
self.sql.update_by_id('experiments', ('exp_id', 'expid7'), {
'status': 'finished',
'end': 1000
})
exp2 = self.db.get_experiment('expid7')
self.assertEqual('expid7', exp2.id)
self.assertEqual('test_experiment', exp2.name)
self.assertEqual('unittest', exp2.creator)
self.assertEqual(100., exp2.start)
self.assertEqual(1000., exp2.end)
self.assertEqual(10., exp2.created)
self.assertEqual('finished', exp2.status)
def test_cancel(self):
self.cctl.register_worker('host1')
exp_id = 'abcd'
exp = Experiment(id=exp_id, name='my-experiment', creator='unittest')
config = ExperimentConfiguration(2, 1, [])
self.ectl.queue(config, exp)
exp = Experiment(id='abcdef',
name='my-experiment-2',
creator='unittest')
config = ExperimentConfiguration(2, 1, [])
self.ectl.queue(config, exp)
self.assertEqual(2, self.ectl.experiment_queue.qsize())
cancelled = self.ectl.cancel(exp_id)
self.assertTrue(cancelled)
self.assertEqual(1, self.ectl.experiment_queue.qsize())
queued = self.ectl.experiment_queue.get()
self.assertEqual('abcdef', queued.experiment.id,
'expected to find experiment that was not cancelled')
def inject_experiment():
exp = Experiment('experiment_id', creator='unittest')
cfg = ExperimentConfiguration(
2, 1,
[WorkloadConfiguration('aservice', [3, 5], 2, 'constant')])
queue = pymq.queue(ExperimentController.queue_key)
queue.put(QueuedExperiment(exp, cfg))
try:
poll(
lambda: queue.qsize() == 0, timeout=2,
interval=0.1) # wait for the daemon to take the experiment
finally:
daemon.close()
def get_running_experiment(self) -> Optional[Experiment]:
fields = [
'exp_id', 'name', 'creator', 'start', 'end', 'created', 'status'
]
fields = self.db.sql_field_list(fields)
sql = f"SELECT {fields} FROM `experiments` WHERE `STATUS` = 'RUNNING'"
entry = self.db.fetchone(sql)
if entry:
row = tuple(entry)
return Experiment(*row)
else:
return None
def test_save_and_get_experiment(self):
expected = Experiment('expid1', 'test_experiment', 'unittest', 10, 100,
1, 'running')
self.db.save_experiment(expected)
actual = self.db.get_experiment('expid1')
self.assertEqual('expid1', actual.id)
self.assertEqual('test_experiment', actual.name)
self.assertEqual('unittest', actual.creator)
self.assertEqual(10., actual.start)
self.assertEqual(100., actual.end)
self.assertEqual(1., actual.created)
self.assertEqual('running', actual.status)
def test_queue(self):
self.cctl.register_worker('host1')
exp = Experiment(name='my-experiment', creator='unittest')
config = ExperimentConfiguration(
2, 1, [WorkloadConfiguration('aservice', [1, 2], 2, 'constant')])
self.ectl.queue(config, exp)
queue = pymq.queue(ExperimentController.queue_key)
self.assertEqual(1, queue.qsize())
self.assertEqual(1, self.ectl.experiment_queue.qsize())
queued_experiment: QueuedExperiment = queue.get()
self.assertEqual(config, queued_experiment.configuration)
self.assertEqual('my-experiment', queued_experiment.experiment.name)
self.assertEqual('unittest', queued_experiment.experiment.creator)
def create_experiment(args):
experiment_id = generate_experiment_id()
if args.name:
name = args.name
else:
name = experiment_id
if args.creator:
creator = args.creator
else:
creator = 'galileodb-recorder-' + str(os.getpid())
now = time.time()
return Experiment(experiment_id,
name=name,
creator=creator,
start=now,
created=now,
status='RUNNING')
def find_all(self) -> List[Experiment]:
sql = 'SELECT * FROM `experiments`'
entries = self.db.fetchall(sql)
return list(map(lambda x: Experiment(*(tuple(x))), entries))
def finalize_experiment(self, exp: Experiment, status):
exp.status = status
exp.end = time.time()
self.update_experiment(exp)
self.touch_traces(exp)
