def set_status(self, status, message=None, traceback=None, **kwargs):
if status in ExperimentLifeCycle.HEARTBEAT_STATUS:
RedisHeartBeat.experiment_ping(self.id)
if ExperimentLifeCycle.can_transition(status_from=self.last_status,
status_to=status):
ExperimentStatus.objects.create(experiment=self,
status=status,
message=message,
traceback=traceback)
def set_status(self, status, created_at=None, message=None, traceback=None, **kwargs):
if status in ExperimentLifeCycle.HEARTBEAT_STATUS:
RedisHeartBeat.experiment_ping(self.id)
last_status = self.last_status_before(status_date=created_at)
if ExperimentLifeCycle.can_transition(status_from=last_status, status_to=status):
params = {'created_at': created_at} if created_at else {}
ExperimentStatus.objects.create(experiment=self,
status=status,
message=message,
traceback=traceback,
**params)
def test_experiments_check_heartbeat(self):
experiment1 = ExperimentFactory()
ExperimentStatusFactory(experiment=experiment1,
status=ExperimentLifeCycle.RUNNING)
RedisHeartBeat.experiment_ping(experiment_id=experiment1.id)
experiment2 = ExperimentFactory()
ExperimentStatusFactory(experiment=experiment2,
status=ExperimentLifeCycle.RUNNING)
experiments_check_heartbeat(experiment1.id)
experiment1.refresh_from_db()
self.assertEqual(experiment1.last_status, ExperimentLifeCycle.RUNNING)
experiments_check_heartbeat(experiment2.id)
experiment2.refresh_from_db()
self.assertEqual(experiment2.last_status, ExperimentLifeCycle.FAILED)
def test_redis_heartbeat_experiment(self):
heartbeat = RedisHeartBeat(experiment=1)
self.assertEqual(heartbeat.redis_key, RedisHeartBeat.KEY_EXPERIMENT.format(1))
self.assertEqual(heartbeat.is_alive(), False)
self.assertEqual(RedisHeartBeat.experiment_is_alive(1), False)
heartbeat.ping()
self.assertEqual(heartbeat.is_alive(), True)
self.assertEqual(RedisHeartBeat.experiment_is_alive(1), True)
heartbeat.clear()
self.assertEqual(heartbeat.is_alive(), False)
self.assertEqual(RedisHeartBeat.experiment_is_alive(1), False)
RedisHeartBeat.experiment_ping(1)
self.assertEqual(heartbeat.is_alive(), True)
self.assertEqual(RedisHeartBeat.experiment_is_alive(1), True)
def post(self, request, *args, **kwargs):
RedisHeartBeat.experiment_ping(experiment_id=self.experiment.id)
return Response(status=status.HTTP_200_OK)
