def test_monitor_experiment_logs(self): expeirment_uuid = uuid.uuid4().hex RedisToStream.monitor_experiment_logs(expeirment_uuid) assert RedisToStream.is_monitored_experiment_logs( expeirment_uuid) is True RedisToStream.remove_experiment_logs(expeirment_uuid) assert RedisToStream.is_monitored_experiment_logs( expeirment_uuid) is False
def test_experiment_monitoring(self): experiment_uuid = uuid.uuid4().hex assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False RedisToStream.monitor_experiment_resources(experiment_uuid) assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is True RedisToStream.remove_experiment_resources(experiment_uuid) assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False RedisToStream.monitor_experiment_logs(experiment_uuid) assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is True RedisToStream.remove_experiment_logs(experiment_uuid) assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False
def test_experiment_monitoring(self): experiment_uuid = uuid.uuid4().hex assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False RedisToStream.monitor_experiment_resources(experiment_uuid) assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is True RedisToStream.remove_experiment_resources(experiment_uuid) assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False RedisToStream.monitor_experiment_logs(experiment_uuid) assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is True RedisToStream.remove_experiment_logs(experiment_uuid) assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False
def publish_experiment_job_log(self, log_line, status, experiment_uuid, experiment_name, job_uuid, task_type=None, task_idx=None): try: log_line = log_line.decode('utf-8') except AttributeError: pass self._logger.info("Publishing log event for task: %s.%s, %s", task_type, task_idx, experiment_name) celery_app.send_task( EventsCeleryTasks.EVENTS_HANDLE_LOGS_EXPERIMENT_JOB, kwargs={ 'experiment_name': experiment_name, 'experiment_uuid': experiment_uuid, 'job_uuid': job_uuid, 'log_line': log_line, 'task_type': task_type, 'task_idx': task_idx }) try: should_stream = ( RedisToStream.is_monitored_job_logs(job_uuid) or RedisToStream.is_monitored_experiment_logs(experiment_uuid)) except RedisError: should_stream = False if should_stream: self._logger.info("Streaming new log event for experiment: %s", experiment_uuid) with celery_app.producer_or_acquire(None) as producer: try: producer.publish( { 'experiment_uuid': experiment_uuid, 'job_uuid': job_uuid, 'log_line': log_line, 'status': status, 'task_type': task_type, 'task_idx': task_idx }, routing_key='{}.{}.{}'.format( RoutingKeys.LOGS_SIDECARS, experiment_uuid, job_uuid), exchange=settings.INTERNAL_EXCHANGE, ) except (TimeoutError, AMQPError): pass
def publish_log(log_line, status, experiment_uuid, experiment_name, job_uuid, task_type=None, task_idx=None): try: log_line = log_line.decode('utf-8') except AttributeError: pass logger.info("Publishing log event for task: {}.{}, {}".format( task_type, task_idx, experiment_name)) handle_events_job_logs.delay(experiment_name=experiment_name, experiment_uuid=experiment_uuid, job_uuid=job_uuid, log_line=log_line, task_type=task_type, task_idx=task_idx) try: should_stream = ( RedisToStream.is_monitored_job_logs(job_uuid) or RedisToStream.is_monitored_experiment_logs(experiment_uuid)) except RedisError: should_stream = False if should_stream: logger.info("Streaming new log event for experiment: {}".format( experiment_uuid)) with celery_app.producer_or_acquire(None) as producer: try: producer.publish( { 'experiment_uuid': experiment_uuid, 'job_uuid': job_uuid, 'log_line': log_line, 'status': status, 'task_type': task_type, 'task_idx': task_idx }, routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS, experiment_uuid, job_uuid), exchange=settings.INTERNAL_EXCHANGE, ) except (TimeoutError, AMQPError): pass
def publish_experiment_job_log(self, log_lines, status, experiment_uuid, experiment_name, job_uuid, task_type=None, task_idx=None): self._logger.debug("Publishing log event for task: %s.%s, %s", task_type, task_idx, experiment_name) celery_app.send_task( EventsCeleryTasks.EVENTS_HANDLE_LOGS_EXPERIMENT_JOB, kwargs={ 'experiment_name': experiment_name, 'experiment_uuid': experiment_uuid, 'job_uuid': job_uuid, 'log_lines': log_lines, 'task_type': task_type, 'task_idx': task_idx}) try: should_stream = (RedisToStream.is_monitored_job_logs(job_uuid) or RedisToStream.is_monitored_experiment_logs(experiment_uuid)) except RedisError: should_stream = False if should_stream: self._logger.info("Streaming new log event for experiment: %s", experiment_uuid) with celery_app.producer_or_acquire(None) as producer: try: producer.publish( { 'experiment_uuid': experiment_uuid, 'job_uuid': job_uuid, 'log_lines': log_lines, 'status': status, 'task_type': task_type, 'task_idx': task_idx }, routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS, experiment_uuid, job_uuid), exchange=settings.INTERNAL_EXCHANGE, ) except (TimeoutError, AMQPError): pass
async def experiment_logs(request, ws, username, project_name, experiment_sequence): project = _get_project(username, project_name) if not has_project_permissions(request.app.user, project, 'GET'): exceptions.Forbidden("You don't have access to this project") experiment = _get_validated_experiment(project, experiment_sequence) experiment_uuid = experiment.uuid.hex auditor.record(event_type=EXPERIMENT_LOGS_VIEWED, instance=experiment, actor_id=request.app.user.id) if not RedisToStream.is_monitored_experiment_logs( experiment_uuid=experiment_uuid): logger.info('Experiment uuid `%s` logs is now being monitored', experiment_uuid) RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid) # start consumer if experiment_uuid in request.app.experiment_logs_consumers: consumer = request.app.experiment_logs_consumers[experiment_uuid] else: logger.info('Add experiment log consumer for %s', experiment_uuid) consumer = Consumer( routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS, experiment_uuid), queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, experiment_uuid)) request.app.experiment_logs_consumers[experiment_uuid] = consumer consumer.run() # add socket manager consumer.add_socket(ws) should_quite = False num_message_retries = 0 while True: num_message_retries += 1 for message in consumer.get_messages(): num_message_retries = 0 disconnected_ws = set() for _ws in consumer.ws: try: await _ws.send(message) except ConnectionClosed: disconnected_ws.add(_ws) consumer.remove_sockets(disconnected_ws) # After trying a couple of time, we must check the status of the experiment if num_message_retries > MAX_RETRIES: experiment.refresh_from_db() if experiment.is_done: logger.info( 'removing all socket because the experiment `%s` is done', experiment_uuid) consumer.ws = set([]) else: num_message_retries -= CHECK_DELAY # Just to check if connection closed if ws._connection_lost: # pylint:disable=protected-access logger.info('Quitting logs socket for experiment uuid %s', experiment_uuid) consumer.remove_sockets({ ws, }) should_quite = True if not consumer.ws: logger.info('Stopping logs monitor for experiment uuid %s', experiment_uuid) RedisToStream.remove_experiment_logs( experiment_uuid=experiment_uuid) # if experiment_uuid in request.app.experiment_logs_consumers: # consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None) # if consumer: # consumer.stop() should_quite = True if should_quite: return await asyncio.sleep(SOCKET_SLEEP)
def test_monitor_experiment_logs(self): expeirment_uuid = uuid.uuid4().hex RedisToStream.monitor_experiment_logs(expeirment_uuid) assert RedisToStream.is_monitored_experiment_logs(expeirment_uuid) is True RedisToStream.remove_experiment_logs(expeirment_uuid) assert RedisToStream.is_monitored_experiment_logs(expeirment_uuid) is False
async def experiment_logs( request, # pylint:disable=too-many-branches ws, username, project_name, experiment_id): experiment, message = validate_experiment(request=request, username=username, project_name=project_name, experiment_id=experiment_id) if experiment is None: await ws.send(get_error_message(message)) return experiment_uuid = experiment.uuid.hex auditor.record(event_type=EXPERIMENT_LOGS_VIEWED, instance=experiment, actor_id=request.app.user.id, actor_name=request.app.user.username) if not RedisToStream.is_monitored_experiment_logs( experiment_uuid=experiment_uuid): _logger.info('Experiment uuid `%s` logs is now being monitored', experiment_uuid) RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid) # start consumer if experiment_uuid in request.app.experiment_logs_consumers: consumer = request.app.experiment_logs_consumers[experiment_uuid] else: _logger.info('Add experiment log consumer for %s', experiment_uuid) consumer = Consumer( routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS_EXPERIMENTS, experiment_uuid), queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, experiment_uuid)) request.app.experiment_logs_consumers[experiment_uuid] = consumer consumer.run() def should_disconnect(): if not consumer.ws: _logger.info('Stopping logs monitor for experiment uuid %s', experiment_uuid) RedisToStream.remove_experiment_logs( experiment_uuid=experiment_uuid) # if experiment_uuid in request.app.experiment_logs_consumers: # consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None) # if consumer: # consumer.stop() return True return False # add socket manager consumer.add_socket(ws) should_quite = False num_message_retries = 0 # Stream phase changes status = None while status != ExperimentLifeCycle.RUNNING and not ExperimentLifeCycle.is_done( status): experiment.refresh_from_db() if status != experiment.last_status: status = experiment.last_status await notify(consumer=consumer, message=get_status_message(status)) if should_disconnect(): return await asyncio.sleep(SOCKET_SLEEP) if ExperimentLifeCycle.is_done(status): await notify(consumer=consumer, message=get_status_message(status)) RedisToStream.remove_experiment_logs(experiment_uuid=experiment_uuid) return while True: num_message_retries += 1 for message in consumer.get_messages(): num_message_retries = 0 await notify(consumer=consumer, message=message) # After trying a couple of time, we must check the status of the experiment if num_message_retries > MAX_RETRIES: experiment.refresh_from_db() if experiment.is_done: _logger.info( 'removing all socket because the experiment `%s` is done', experiment_uuid) consumer.ws = set([]) else: num_message_retries -= CHECK_DELAY # Just to check if connection closed if ws._connection_lost: # pylint:disable=protected-access _logger.info('Quitting logs socket for experiment uuid %s', experiment_uuid) consumer.remove_sockets({ ws, }) should_quite = True if should_disconnect(): should_quite = True if should_quite: return await asyncio.sleep(SOCKET_SLEEP)
async def experiment_logs(request, ws, username, project_name, experiment_sequence): project = _get_project(username, project_name) if not has_project_permissions(request.app.user, project, 'GET'): exceptions.Forbidden("You don't have access to this project") experiment = _get_validated_experiment(project, experiment_sequence) experiment_uuid = experiment.uuid.hex auditor.record(event_type=EXPERIMENT_LOGS_VIEWED, instance=experiment, actor_id=request.app.user.id) if not RedisToStream.is_monitored_experiment_logs(experiment_uuid=experiment_uuid): logger.info('Experiment uuid `%s` logs is now being monitored', experiment_uuid) RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid) # start consumer if experiment_uuid in request.app.experiment_logs_consumers: consumer = request.app.experiment_logs_consumers[experiment_uuid] else: logger.info('Add experiment log consumer for %s', experiment_uuid) consumer = Consumer( routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS, experiment_uuid), queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, experiment_uuid)) request.app.experiment_logs_consumers[experiment_uuid] = consumer consumer.run() # add socket manager consumer.add_socket(ws) should_quite = False num_message_retries = 0 while True: num_message_retries += 1 for message in consumer.get_messages(): num_message_retries = 0 disconnected_ws = set() for _ws in consumer.ws: try: await _ws.send(message) except ConnectionClosed: disconnected_ws.add(_ws) consumer.remove_sockets(disconnected_ws) # After trying a couple of time, we must check the status of the experiment if num_message_retries > MAX_RETRIES: experiment.refresh_from_db() if experiment.is_done: logger.info( 'removing all socket because the experiment `%s` is done', experiment_uuid) consumer.ws = set([]) else: num_message_retries -= CHECK_DELAY # Just to check if connection closed if ws._connection_lost: # pylint:disable=protected-access logger.info('Quitting logs socket for experiment uuid %s', experiment_uuid) consumer.remove_sockets({ws, }) should_quite = True if not consumer.ws: logger.info('Stopping logs monitor for experiment uuid %s', experiment_uuid) RedisToStream.remove_experiment_logs(experiment_uuid=experiment_uuid) # if experiment_uuid in request.app.experiment_logs_consumers: # consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None) # if consumer: # consumer.stop() should_quite = True if should_quite: return await asyncio.sleep(SOCKET_SLEEP)