def test_bo_rescheduling(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as mock_fct:
ExperimentGroupFactory(content=experiment_group_spec_content_bo)
assert mock_fct.call_count == 2
with patch.object(GroupChecks, 'is_checked') as mock_is_check:
with patch('hpsearch.tasks.bo.hp_bo_iterate.apply_async'
) as mock_fct1:
with patch('scheduler.tasks.experiments.'
'experiments_build.apply_async') as mock_fct2:
mock_is_check.return_value = False
ExperimentGroupFactory(
content=experiment_group_spec_content_bo)
assert mock_fct1.call_count == 2
# 2 experiments, but since we are mocking the scheduling function, it's 4 calls,
# every call to start tries to schedule again, but in reality it's just 2 calls
assert mock_fct2.call_count == 4
# Fake
with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_bo)
assert mock_fct.call_count == 2
assert experiment_group.non_done_experiments.count() == 2
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
with patch('hpsearch.tasks.bo.hp_bo_start.apply_async'
) as xp_trigger_start:
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(
experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
assert xp_trigger_start.call_count == experiment_group.experiments.count(
)
with patch('hpsearch.tasks.bo.hp_bo_iterate.apply_async') as mock_fct1:
hp_bo_start(experiment_group.id)
assert mock_fct1.call_count == 1
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
with patch('hpsearch.tasks.bo.hp_bo_start.apply_async'
) as xp_trigger_start:
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(
experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
assert xp_trigger_start.call_count == experiment_group.experiments.count(
)
GroupChecks(group=experiment_group.id).clear()
with patch('hpsearch.tasks.bo.hp_bo_create.apply_async') as mock_fct1:
hp_bo_start(experiment_group.id)
assert mock_fct1.call_count == 1
def should_group_start(experiment_group_id, task, auto_retry):
group_checks = GroupChecks(group=experiment_group_id)
if group_checks.is_delayed():
return False
if group_checks.is_checked():
group_checks.delay()
celery_app.send_task(
task,
kwargs={'experiment_group_id': experiment_group_id, 'auto_retry': auto_retry},
countdown=conf.get('GROUP_CHECKS_INTERVAL'))
return False
group_checks.check()
return True
def should_group_start(experiment_group_id, task, auto_retry):
group_checks = GroupChecks(group=experiment_group_id)
if group_checks.is_delayed():
return False
if group_checks.is_checked():
group_checks.delay()
workers.send(task,
kwargs={
'experiment_group_id': experiment_group_id,
'auto_retry': auto_retry
},
countdown=conf.get(GROUPS_CHECK_INTERVAL))
return False
group_checks.check()
return True
def test_redis_group_checks(self):
group_checks = GroupChecks(group=1)
self.assertEqual(group_checks.redis_key_checked,
GroupChecks.KEY_CHECKED.format(1))
self.assertEqual(group_checks.redis_key_delayed,
GroupChecks.KEY_DELAYED.format(1))
self.assertEqual(group_checks.is_checked(), False)
self.assertEqual(group_checks.is_delayed(), False)
group_checks.check()
self.assertEqual(group_checks.is_checked(), True)
self.assertEqual(group_checks.is_delayed(), False)
group_checks.clear()
self.assertEqual(group_checks.is_checked(), False)
self.assertEqual(group_checks.is_delayed(), False)
group_checks.delay()
self.assertEqual(group_checks.is_checked(), True)
self.assertEqual(group_checks.is_delayed(), True)
group_checks.clear()
self.assertEqual(group_checks.is_checked(), False)
self.assertEqual(group_checks.is_delayed(), False)
group_checks.check()
self.assertEqual(group_checks.is_checked(), True)
self.assertEqual(group_checks.is_delayed(), False)
group_checks.delay()
self.assertEqual(group_checks.is_checked(), True)
self.assertEqual(group_checks.is_delayed(), True)