def test_monitor_experiment_logs(self):
expeirment_uuid = uuid.uuid4().hex
RedisToStream.monitor_experiment_logs(expeirment_uuid)
assert RedisToStream.is_monitored_experiment_logs(
expeirment_uuid) is True
RedisToStream.remove_experiment_logs(expeirment_uuid)
assert RedisToStream.is_monitored_experiment_logs(
expeirment_uuid) is False
def test_experiment_monitoring(self):
experiment_uuid = uuid.uuid4().hex
assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False
RedisToStream.monitor_experiment_resources(experiment_uuid)
assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is True
RedisToStream.remove_experiment_resources(experiment_uuid)
assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False
assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False
RedisToStream.monitor_experiment_logs(experiment_uuid)
assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is True
RedisToStream.remove_experiment_logs(experiment_uuid)
assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False
def test_experiment_monitoring(self):
experiment_uuid = uuid.uuid4().hex
assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False
RedisToStream.monitor_experiment_resources(experiment_uuid)
assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is True
RedisToStream.remove_experiment_resources(experiment_uuid)
assert RedisToStream.is_monitored_experiment_resources(experiment_uuid) is False
assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False
RedisToStream.monitor_experiment_logs(experiment_uuid)
assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is True
RedisToStream.remove_experiment_logs(experiment_uuid)
assert RedisToStream.is_monitored_experiment_logs(experiment_uuid) is False
def publish_experiment_job_log(self,
log_line,
status,
experiment_uuid,
experiment_name,
job_uuid,
task_type=None,
task_idx=None):
try:
log_line = log_line.decode('utf-8')
except AttributeError:
pass
self._logger.info("Publishing log event for task: %s.%s, %s",
task_type, task_idx, experiment_name)
celery_app.send_task(
EventsCeleryTasks.EVENTS_HANDLE_LOGS_EXPERIMENT_JOB,
kwargs={
'experiment_name': experiment_name,
'experiment_uuid': experiment_uuid,
'job_uuid': job_uuid,
'log_line': log_line,
'task_type': task_type,
'task_idx': task_idx
})
try:
should_stream = (
RedisToStream.is_monitored_job_logs(job_uuid)
or RedisToStream.is_monitored_experiment_logs(experiment_uuid))
except RedisError:
should_stream = False
if should_stream:
self._logger.info("Streaming new log event for experiment: %s",
experiment_uuid)
with celery_app.producer_or_acquire(None) as producer:
try:
producer.publish(
{
'experiment_uuid': experiment_uuid,
'job_uuid': job_uuid,
'log_line': log_line,
'status': status,
'task_type': task_type,
'task_idx': task_idx
},
routing_key='{}.{}.{}'.format(
RoutingKeys.LOGS_SIDECARS, experiment_uuid,
job_uuid),
exchange=settings.INTERNAL_EXCHANGE,
)
except (TimeoutError, AMQPError):
pass
def publish_log(log_line,
status,
experiment_uuid,
experiment_name,
job_uuid,
task_type=None,
task_idx=None):
try:
log_line = log_line.decode('utf-8')
except AttributeError:
pass
logger.info("Publishing log event for task: {}.{}, {}".format(
task_type, task_idx, experiment_name))
handle_events_job_logs.delay(experiment_name=experiment_name,
experiment_uuid=experiment_uuid,
job_uuid=job_uuid,
log_line=log_line,
task_type=task_type,
task_idx=task_idx)
try:
should_stream = (
RedisToStream.is_monitored_job_logs(job_uuid)
or RedisToStream.is_monitored_experiment_logs(experiment_uuid))
except RedisError:
should_stream = False
if should_stream:
logger.info("Streaming new log event for experiment: {}".format(
experiment_uuid))
with celery_app.producer_or_acquire(None) as producer:
try:
producer.publish(
{
'experiment_uuid': experiment_uuid,
'job_uuid': job_uuid,
'log_line': log_line,
'status': status,
'task_type': task_type,
'task_idx': task_idx
},
routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS,
experiment_uuid, job_uuid),
exchange=settings.INTERNAL_EXCHANGE,
)
except (TimeoutError, AMQPError):
pass
def publish_experiment_job_log(self,
log_lines,
status,
experiment_uuid,
experiment_name,
job_uuid,
task_type=None,
task_idx=None):
self._logger.debug("Publishing log event for task: %s.%s, %s",
task_type, task_idx, experiment_name)
celery_app.send_task(
EventsCeleryTasks.EVENTS_HANDLE_LOGS_EXPERIMENT_JOB,
kwargs={
'experiment_name': experiment_name,
'experiment_uuid': experiment_uuid,
'job_uuid': job_uuid,
'log_lines': log_lines,
'task_type': task_type,
'task_idx': task_idx})
try:
should_stream = (RedisToStream.is_monitored_job_logs(job_uuid) or
RedisToStream.is_monitored_experiment_logs(experiment_uuid))
except RedisError:
should_stream = False
if should_stream:
self._logger.info("Streaming new log event for experiment: %s", experiment_uuid)
with celery_app.producer_or_acquire(None) as producer:
try:
producer.publish(
{
'experiment_uuid': experiment_uuid,
'job_uuid': job_uuid,
'log_lines': log_lines,
'status': status,
'task_type': task_type,
'task_idx': task_idx
},
routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS,
experiment_uuid,
job_uuid),
exchange=settings.INTERNAL_EXCHANGE,
)
except (TimeoutError, AMQPError):
pass
async def experiment_logs(request, ws, username, project_name,
experiment_sequence):
project = _get_project(username, project_name)
if not has_project_permissions(request.app.user, project, 'GET'):
exceptions.Forbidden("You don't have access to this project")
experiment = _get_validated_experiment(project, experiment_sequence)
experiment_uuid = experiment.uuid.hex
auditor.record(event_type=EXPERIMENT_LOGS_VIEWED,
instance=experiment,
actor_id=request.app.user.id)
if not RedisToStream.is_monitored_experiment_logs(
experiment_uuid=experiment_uuid):
logger.info('Experiment uuid `%s` logs is now being monitored',
experiment_uuid)
RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid)
# start consumer
if experiment_uuid in request.app.experiment_logs_consumers:
consumer = request.app.experiment_logs_consumers[experiment_uuid]
else:
logger.info('Add experiment log consumer for %s', experiment_uuid)
consumer = Consumer(
routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS,
experiment_uuid),
queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS,
experiment_uuid))
request.app.experiment_logs_consumers[experiment_uuid] = consumer
consumer.run()
# add socket manager
consumer.add_socket(ws)
should_quite = False
num_message_retries = 0
while True:
num_message_retries += 1
for message in consumer.get_messages():
num_message_retries = 0
disconnected_ws = set()
for _ws in consumer.ws:
try:
await _ws.send(message)
except ConnectionClosed:
disconnected_ws.add(_ws)
consumer.remove_sockets(disconnected_ws)
# After trying a couple of time, we must check the status of the experiment
if num_message_retries > MAX_RETRIES:
experiment.refresh_from_db()
if experiment.is_done:
logger.info(
'removing all socket because the experiment `%s` is done',
experiment_uuid)
consumer.ws = set([])
else:
num_message_retries -= CHECK_DELAY
# Just to check if connection closed
if ws._connection_lost: # pylint:disable=protected-access
logger.info('Quitting logs socket for experiment uuid %s',
experiment_uuid)
consumer.remove_sockets({
ws,
})
should_quite = True
if not consumer.ws:
logger.info('Stopping logs monitor for experiment uuid %s',
experiment_uuid)
RedisToStream.remove_experiment_logs(
experiment_uuid=experiment_uuid)
# if experiment_uuid in request.app.experiment_logs_consumers:
# consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None)
# if consumer:
# consumer.stop()
should_quite = True
if should_quite:
return
await asyncio.sleep(SOCKET_SLEEP)
def test_monitor_experiment_logs(self):
expeirment_uuid = uuid.uuid4().hex
RedisToStream.monitor_experiment_logs(expeirment_uuid)
assert RedisToStream.is_monitored_experiment_logs(expeirment_uuid) is True
RedisToStream.remove_experiment_logs(expeirment_uuid)
assert RedisToStream.is_monitored_experiment_logs(expeirment_uuid) is False
async def experiment_logs(
request, # pylint:disable=too-many-branches
ws,
username,
project_name,
experiment_id):
experiment, message = validate_experiment(request=request,
username=username,
project_name=project_name,
experiment_id=experiment_id)
if experiment is None:
await ws.send(get_error_message(message))
return
experiment_uuid = experiment.uuid.hex
auditor.record(event_type=EXPERIMENT_LOGS_VIEWED,
instance=experiment,
actor_id=request.app.user.id,
actor_name=request.app.user.username)
if not RedisToStream.is_monitored_experiment_logs(
experiment_uuid=experiment_uuid):
_logger.info('Experiment uuid `%s` logs is now being monitored',
experiment_uuid)
RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid)
# start consumer
if experiment_uuid in request.app.experiment_logs_consumers:
consumer = request.app.experiment_logs_consumers[experiment_uuid]
else:
_logger.info('Add experiment log consumer for %s', experiment_uuid)
consumer = Consumer(
routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS_EXPERIMENTS,
experiment_uuid),
queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS,
experiment_uuid))
request.app.experiment_logs_consumers[experiment_uuid] = consumer
consumer.run()
def should_disconnect():
if not consumer.ws:
_logger.info('Stopping logs monitor for experiment uuid %s',
experiment_uuid)
RedisToStream.remove_experiment_logs(
experiment_uuid=experiment_uuid)
# if experiment_uuid in request.app.experiment_logs_consumers:
# consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None)
# if consumer:
# consumer.stop()
return True
return False
# add socket manager
consumer.add_socket(ws)
should_quite = False
num_message_retries = 0
# Stream phase changes
status = None
while status != ExperimentLifeCycle.RUNNING and not ExperimentLifeCycle.is_done(
status):
experiment.refresh_from_db()
if status != experiment.last_status:
status = experiment.last_status
await notify(consumer=consumer, message=get_status_message(status))
if should_disconnect():
return
await asyncio.sleep(SOCKET_SLEEP)
if ExperimentLifeCycle.is_done(status):
await notify(consumer=consumer, message=get_status_message(status))
RedisToStream.remove_experiment_logs(experiment_uuid=experiment_uuid)
return
while True:
num_message_retries += 1
for message in consumer.get_messages():
num_message_retries = 0
await notify(consumer=consumer, message=message)
# After trying a couple of time, we must check the status of the experiment
if num_message_retries > MAX_RETRIES:
experiment.refresh_from_db()
if experiment.is_done:
_logger.info(
'removing all socket because the experiment `%s` is done',
experiment_uuid)
consumer.ws = set([])
else:
num_message_retries -= CHECK_DELAY
# Just to check if connection closed
if ws._connection_lost: # pylint:disable=protected-access
_logger.info('Quitting logs socket for experiment uuid %s',
experiment_uuid)
consumer.remove_sockets({
ws,
})
should_quite = True
if should_disconnect():
should_quite = True
if should_quite:
return
await asyncio.sleep(SOCKET_SLEEP)
async def experiment_logs(request, ws, username, project_name, experiment_sequence):
project = _get_project(username, project_name)
if not has_project_permissions(request.app.user, project, 'GET'):
exceptions.Forbidden("You don't have access to this project")
experiment = _get_validated_experiment(project, experiment_sequence)
experiment_uuid = experiment.uuid.hex
auditor.record(event_type=EXPERIMENT_LOGS_VIEWED,
instance=experiment,
actor_id=request.app.user.id)
if not RedisToStream.is_monitored_experiment_logs(experiment_uuid=experiment_uuid):
logger.info('Experiment uuid `%s` logs is now being monitored', experiment_uuid)
RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid)
# start consumer
if experiment_uuid in request.app.experiment_logs_consumers:
consumer = request.app.experiment_logs_consumers[experiment_uuid]
else:
logger.info('Add experiment log consumer for %s', experiment_uuid)
consumer = Consumer(
routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS, experiment_uuid),
queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, experiment_uuid))
request.app.experiment_logs_consumers[experiment_uuid] = consumer
consumer.run()
# add socket manager
consumer.add_socket(ws)
should_quite = False
num_message_retries = 0
while True:
num_message_retries += 1
for message in consumer.get_messages():
num_message_retries = 0
disconnected_ws = set()
for _ws in consumer.ws:
try:
await _ws.send(message)
except ConnectionClosed:
disconnected_ws.add(_ws)
consumer.remove_sockets(disconnected_ws)
# After trying a couple of time, we must check the status of the experiment
if num_message_retries > MAX_RETRIES:
experiment.refresh_from_db()
if experiment.is_done:
logger.info(
'removing all socket because the experiment `%s` is done', experiment_uuid)
consumer.ws = set([])
else:
num_message_retries -= CHECK_DELAY
# Just to check if connection closed
if ws._connection_lost: # pylint:disable=protected-access
logger.info('Quitting logs socket for experiment uuid %s', experiment_uuid)
consumer.remove_sockets({ws, })
should_quite = True
if not consumer.ws:
logger.info('Stopping logs monitor for experiment uuid %s', experiment_uuid)
RedisToStream.remove_experiment_logs(experiment_uuid=experiment_uuid)
# if experiment_uuid in request.app.experiment_logs_consumers:
# consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None)
# if consumer:
# consumer.stop()
should_quite = True
if should_quite:
return
await asyncio.sleep(SOCKET_SLEEP)