def test_master_success_influences_other_experiment_workers_status(self): with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiment = ExperimentFactory() assert ExperimentLifeCycle.is_done(experiment.last_status) is False # Add jobs master = ExperimentJobFactory(experiment=experiment, role=TaskType.MASTER) assert JobLifeCycle.is_done(master.last_status) is False workers = [ ExperimentJobFactory(experiment=experiment, role=TaskType.WORKER) for _ in range(2) ] for worker in workers: worker.refresh_from_db() assert JobLifeCycle.is_done(worker.last_status) is False # Set master to succeeded ExperimentJobStatusFactory(job=master, status=JobLifeCycle.SUCCEEDED) # All worker should have a success status for worker in workers: worker.refresh_from_db() assert worker.last_status == JobLifeCycle.SUCCEEDED # Experiment last status should be success experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.SUCCEEDED
def build_experiment(self, experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: if self.request.retries < 2: _logger.info('Trying again for Experiment `%s`.', experiment_id) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) _logger.info( 'Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_id) return # No need to build the image, start the experiment directly if not (experiment.specification.build and experiment.specification.run): celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}) return if not ExperimentLifeCycle.can_transition( status_from=experiment.last_status, status_to=ExperimentLifeCycle.BUILDING): _logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.', experiment_id, experiment.last_status, ExperimentLifeCycle.BUILDING) return None # Update experiment status to show that its building experiment.set_status(ExperimentLifeCycle.BUILDING) # Building the docker image try: status = experiments_builder.build_experiment(experiment) except DockerException as e: _logger.warning('Failed to build experiment %s', e) experiment.set_status(ExperimentLifeCycle.FAILED, message='Failed to build image for experiment.') return except Repo.DoesNotExist: _logger.warning('No code was found for this project') experiment.set_status( ExperimentLifeCycle.FAILED, message='No code was found for to build this experiment.') return except Exception as e: # Other exceptions _logger.error( 'Failed to build experiment, unexpected error occurred.\n%s', traceback.format_exc()) experiment.set_status(ExperimentLifeCycle.FAILED, message='Failed to build image for experiment.') return if not status: experiment.set_status(ExperimentLifeCycle.FAILED, message='Failed to build image for experiment.') return # Now we can start the experiment celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id})
def calculated_status(self): master_status = self.jobs.filter(role=TaskType.MASTER)[0].last_status calculated_status = master_status if JobLifeCycle.is_done(master_status) else None if calculated_status is None: calculated_status = ExperimentLifeCycle.jobs_status(self.last_job_statuses) if calculated_status is None: return self.last_status return calculated_status
def calculated_status(self): master_status = self.jobs.filter(role=TaskType.MASTER)[0].last_status calculated_status = master_status if JobLifeCycle.is_done(master_status) else None if calculated_status is None: calculated_status = ExperimentLifeCycle.jobs_status(self.last_job_statuses) if calculated_status is None: return self.last_status return calculated_status
def post(self, request, *args, **kwargs): experiment = self.get_object() if not ExperimentLifeCycle.is_stoppable(experiment.last_status): return Response(status=status.HTTP_403_FORBIDDEN) token, _ = Token.objects.get_or_create(user=experiment.user) return Response({'token': token.key}, status=status.HTTP_200_OK)
def set_status(self, status, message=None, traceback=None, **kwargs): if status in ExperimentLifeCycle.HEARTBEAT_STATUS: RedisHeartBeat.experiment_ping(self.id) if ExperimentLifeCycle.can_transition(status_from=self.last_status, status_to=status): ExperimentStatus.objects.create(experiment=self, status=status, message=message, traceback=traceback)
def calculated_status(self) -> str: master_status = self.jobs.order_by('created_at').first().last_status calculated_status = master_status if JobLifeCycle.is_done( master_status) else None if calculated_status is None: calculated_status = ExperimentLifeCycle.jobs_status( self.last_job_statuses) if calculated_status is None: return self.last_status return calculated_status
def set_status(self, status, created_at=None, message=None, traceback=None, **kwargs): if status in ExperimentLifeCycle.HEARTBEAT_STATUS: RedisHeartBeat.experiment_ping(self.id) last_status = self.last_status_before(status_date=created_at) if ExperimentLifeCycle.can_transition(status_from=last_status, status_to=status): params = {'created_at': created_at} if created_at else {} ExperimentStatus.objects.create(experiment=self, status=status, message=message, traceback=traceback, **params)
async def log_experiment(request, ws, experiment, namespace, container): experiment_uuid = experiment.uuid.hex if experiment_uuid in request.app.experiment_logs_ws_managers: ws_manager = request.app.experiment_logs_ws_managers[experiment_uuid] else: ws_manager = SocketManager() request.app.experiment_logs_ws_managers[experiment_uuid] = ws_manager ws_manager.add_socket(ws) # Stream phase changes status = None while status != ExperimentLifeCycle.RUNNING and not ExperimentLifeCycle.is_done( status): experiment.refresh_from_db() if status != experiment.last_status: status = experiment.last_status await notify_ws(ws=ws, message=get_status_message(status)) if should_disconnect(ws=ws, ws_manager=ws_manager): return await asyncio.sleep(SOCKET_SLEEP) if ExperimentLifeCycle.is_done(status): await notify_ws(ws=ws, message=get_status_message(status)) return config.load_incluster_config() k8s_api = client.CoreV1Api() log_requests = [] for job in experiment.jobs.all(): pod_id = job.pod_id log_requests.append( log_job_pod(k8s_api=k8s_api, ws=ws, ws_manager=ws_manager, pod_id=pod_id, container=container, namespace=namespace, task_type=job.role, task_idx=job.sequence)) await asyncio.wait(log_requests)
def build_experiment(self, experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: if self.request.retries < 2: _logger.info('Trying again for Experiment `%s`.', experiment_id) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) _logger.info('Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_id) return # No need to build the image, start the experiment directly if not (experiment.specification.build and experiment.specification.run): celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}) return if not ExperimentLifeCycle.can_transition(status_from=experiment.last_status, status_to=ExperimentLifeCycle.BUILDING): _logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.', experiment_id, experiment.last_status, ExperimentLifeCycle.BUILDING) return None # Update experiment status to show that its building experiment.set_status(ExperimentLifeCycle.BUILDING) # Building the docker image try: status = experiments_builder.build_experiment(experiment) except DockerException as e: _logger.warning('Failed to build experiment %s', e) experiment.set_status(ExperimentLifeCycle.FAILED, message='Failed to build image for experiment.') return except Repo.DoesNotExist: _logger.warning('No code was found for this project') experiment.set_status(ExperimentLifeCycle.FAILED, message='No code was found for to build this experiment.') return except Exception as e: # Other exceptions _logger.warning('Failed to build experiment %s', e) experiment.set_status(ExperimentLifeCycle.FAILED, message='Failed to build image for experiment.') return if not status: return # Now we can start the experiment celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id})
def experiment_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment previous_status = experiment.last_status # update experiment last_status experiment.status = instance set_started_at(instance=experiment, status=instance.status, starting_statuses=[ ExperimentLifeCycle.STARTING, ExperimentLifeCycle.RUNNING ], running_status=ExperimentLifeCycle.RUNNING) set_finished_at(instance=experiment, status=instance.status, is_done=ExperimentLifeCycle.is_done) experiment.save(update_fields=['status', 'started_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_NEW_STATUS, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.SUCCEEDED: # update all workers with succeeded status, since we will trigger a stop mechanism for job in experiment.jobs.all(): if not job.is_done: job.set_status(JobLifeCycle.SUCCEEDED, message='Master is done.') auditor.record(event_type=EXPERIMENT_SUCCEEDED, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.FAILED: auditor.record(event_type=EXPERIMENT_FAILED, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_STOPPED, instance=experiment, previous_status=previous_status) if ExperimentLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_DONE, instance=experiment, previous_status=previous_status) # Check if it's part of an experiment group, and start following tasks if not experiment.is_independent: celery_app.send_task( HPCeleryTasks.HP_START, kwargs={'experiment_group_id': experiment.experiment_group.id}, countdown=1)
def experiments_start(experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: _logger.info('Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_id) return if not ExperimentLifeCycle.can_transition(status_from=experiment.last_status, status_to=ExperimentLifeCycle.SCHEDULED): _logger.info('Experiment `%s` cannot transition from `%s` to `%s`.', experiment.unique_name, experiment.last_status, ExperimentLifeCycle.SCHEDULED) return None experiment_scheduler.start_experiment(experiment)
def experiment_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment previous_status = experiment.last_status # update experiment last_status experiment.status = instance set_started_at(instance=experiment, status=instance.status, starting_statuses=[ ExperimentLifeCycle.STARTING, ExperimentLifeCycle.RUNNING ], running_status=ExperimentLifeCycle.RUNNING) set_finished_at(instance=experiment, status=instance.status, is_done=ExperimentLifeCycle.is_done) experiment.save( update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_NEW_STATUS, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.CREATED: auditor.record(event_type=EXPERIMENT_CREATED, instance=experiment) elif instance.status == ExperimentLifeCycle.SUCCEEDED: # update all workers with succeeded status, since we will trigger a stop mechanism for job in experiment.jobs.all(): if not job.is_done: job.set_status(JobLifeCycle.SUCCEEDED, message='Master is done.') auditor.record(event_type=EXPERIMENT_SUCCEEDED, instance=experiment, previous_status=previous_status) elif instance.status == ExperimentLifeCycle.FAILED: auditor.record(event_type=EXPERIMENT_FAILED, instance=experiment, previous_status=previous_status) elif instance.status == ExperimentLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_STOPPED, instance=experiment, previous_status=previous_status) if ExperimentLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_DONE, instance=experiment, previous_status=previous_status)
def experiments_build(experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: return # No need to build the image, start the experiment directly if not (experiment.specification.build and experiment.specification.run): celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}, countdown=conf.get('GLOBAL_COUNTDOWN')) return last_status = experiment.last_status if not ExperimentLifeCycle.can_transition( status_from=last_status, status_to=ExperimentLifeCycle.BUILDING): _logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.', experiment_id, last_status, ExperimentLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=experiment.user, project=experiment.project, config=experiment.specification.build, configmap_refs=experiment.specification.configmap_refs, secret_refs=experiment.specification.secret_refs, code_reference=experiment.code_reference) experiment.build_job = build_job experiment.save(update_fields=['build_job']) if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}, countdown=conf.get('GLOBAL_COUNTDOWN')) return if not build_status: experiment.set_status(ExperimentLifeCycle.FAILED, message='Could not start build process.') return # Update experiment status to show that its building experiment.set_status(ExperimentLifeCycle.BUILDING)
def experiments_start(experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: _logger.info('Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_id) return if not ExperimentLifeCycle.can_transition(status_from=experiment.last_status, status_to=ExperimentLifeCycle.SCHEDULED): _logger.info('Experiment `%s` cannot transition from `%s` to `%s`.', experiment.unique_name, experiment.last_status, ExperimentLifeCycle.SCHEDULED) return None # Check if we need to copy an experiment if experiment.is_copy: copy_experiment(experiment) else: create_experiment_outputs_path(experiment.unique_name) experiment_scheduler.start_experiment(experiment)
def experiments_build(experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: return # No need to build the image, start the experiment directly if not (experiment.specification.build and experiment.specification.run): celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}) return if not ExperimentLifeCycle.can_transition(status_from=experiment.last_status, status_to=ExperimentLifeCycle.BUILDING): _logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.', experiment_id, experiment.last_status, ExperimentLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=experiment.user, project=experiment.project, config=experiment.specification.build, code_reference=experiment.code_reference) experiment.build_job = build_job experiment.save() if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}) return if not build_status: experiment.set_status(ExperimentLifeCycle.FAILED, message='Could not start build process.') return # Update experiment status to show that its building experiment.set_status(ExperimentLifeCycle.BUILDING)
async def experiment_logs( request, # pylint:disable=too-many-branches ws, username, project_name, experiment_id): from streams.consumers.consumers import Consumer experiment, message = validate_experiment(request=request, username=username, project_name=project_name, experiment_id=experiment_id) if experiment is None: await ws.send(get_error_message(message)) return experiment_uuid = experiment.uuid.hex auditor.record(event_type=EXPERIMENT_LOGS_VIEWED, instance=experiment, actor_id=request.app.user.id, actor_name=request.app.user.username) if not RedisToStream.is_monitored_experiment_logs( experiment_uuid=experiment_uuid): logger.info('Experiment uuid `%s` logs is now being monitored', experiment_uuid) RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid) # start consumer if experiment_uuid in request.app.experiment_logs_consumers: consumer = request.app.experiment_logs_consumers[experiment_uuid] else: logger.info('Add experiment log consumer for %s', experiment_uuid) consumer = Consumer(routing_key='{}.{}.*'.format( RoutingKeys.STREAM_LOGS_SIDECARS_EXPERIMENTS, experiment_uuid), queue='{}.{}'.format( CeleryQueues.STREAM_LOGS_SIDECARS, experiment_uuid)) request.app.experiment_logs_consumers[experiment_uuid] = consumer consumer.run() def should_disconnect(): if not consumer.ws: logger.info('Stopping logs monitor for experiment uuid %s', experiment_uuid) RedisToStream.remove_experiment_logs( experiment_uuid=experiment_uuid) # if experiment_uuid in request.app.experiment_logs_consumers: # consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None) # if consumer: # consumer.stop() return True return False # add socket manager consumer.add_socket(ws) should_quite = False num_message_retries = 0 # Stream phase changes status = None while status != ExperimentLifeCycle.RUNNING and not ExperimentLifeCycle.is_done( status): experiment.refresh_from_db() if status != experiment.last_status: status = experiment.last_status await notify(ws_manager=consumer, message=get_status_message(status)) if should_disconnect(): return await asyncio.sleep(SOCKET_SLEEP) if ExperimentLifeCycle.is_done(status): await notify(ws_manager=consumer, message=get_status_message(status)) RedisToStream.remove_experiment_logs(experiment_uuid=experiment_uuid) return while True: num_message_retries += 1 for message in consumer.get_messages(): num_message_retries = 0 await notify(ws_manager=consumer, message=message) # After trying a couple of time, we must check the status of the experiment if num_message_retries > MAX_RETRIES: experiment.refresh_from_db() if experiment.is_done: logger.info( 'removing all socket because the experiment `%s` is done', experiment_uuid) consumer.ws = set([]) else: num_message_retries -= CHECK_DELAY # Just to check if connection closed if ws._connection_lost: # pylint:disable=protected-access logger.info('Quitting logs socket for experiment uuid %s', experiment_uuid) consumer.remove_sockets({ ws, }) should_quite = True if should_disconnect(): should_quite = True if should_quite: return await asyncio.sleep(SOCKET_SLEEP)