def handle_new_experiment_status(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment if not experiment.specification: return stop_condition = (instance.status in (ExperimentLifeCycle.FAILED, ExperimentLifeCycle.SUCCEEDED) and experiment.jobs.count() > 0) if stop_condition: _logger.info( 'One of the workers failed or Master for experiment `%s` is done, ' 'send signal to other workers to stop.', experiment.unique_name) # Schedule stop for this experiment because other jobs may be still running group = experiment.experiment_group celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_STOP, kwargs={ 'project_name': experiment.project.unique_name, 'project_uuid': experiment.project.uuid.hex, 'experiment_name': experiment.unique_name, 'experiment_uuid': experiment.uuid.hex, 'experiment_group_name': group.unique_name if group else None, 'experiment_group_uuid': group.uuid.hex if group else None, 'specification': experiment.config, 'update_status': False }, countdown=RedisTTL.get_for_experiment(experiment_id=experiment.id))
def _handle_experiment_post_run(cls, event: 'Event') -> None: instance = event.instance if not instance or not instance.has_specification or not instance.jobs.count() > 0: return # Schedule stop for this experiment because other jobs may be still running group = instance.experiment_group celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_STOP, kwargs={ 'project_name': instance.project.unique_name, 'project_uuid': instance.project.uuid.hex, 'experiment_name': instance.unique_name, 'experiment_uuid': instance.uuid.hex, 'experiment_group_name': group.unique_name if group else None, 'experiment_group_uuid': group.uuid.hex if group else None, 'specification': instance.config, 'update_status': False, 'collect_logs': True, }, countdown=RedisTTL.get_for_experiment(experiment_id=instance.id))
def test_set_for_experiment(self): RedisTTL.set_for_experiment(experiment_id=1, value=10) assert RedisTTL.get_for_experiment(experiment_id=1) == 10 assert RedisTTL.get_for_experiment( experiment_id=2) == conf.get('GLOBAL_COUNTDOWN') assert RedisTTL(experiment=10).get_value() is None
def test_set_for_experiment(self): RedisTTL.set_for_experiment(experiment_id=1, value=10) assert RedisTTL.get_for_experiment(experiment_id=1) == 10 assert RedisTTL.get_for_experiment(experiment_id=2) == 2 assert RedisTTL(experiment=10).get_value() is None