async def job_resources(request, ws, username, project_name, experiment_sequence, job_sequence):
project = _get_project(username, project_name)
if not has_project_permissions(request.app.user, project, 'GET'):
exceptions.Forbidden("You don't have access to this project")
experiment = _get_validated_experiment(project, experiment_sequence)
job = _get_job(experiment, job_sequence)
job_uuid = job.uuid.hex
job_name = '{}.{}'.format(job.role, job.sequence)
auditor.record(event_type=EXPERIMENT_JOB_RESOURCES_VIEWED,
instance=job,
actor_id=request.app.user.id)
if not RedisToStream.is_monitored_job_resources(job_uuid=job_uuid):
_logger.info('Job resources with uuid `%s` is now being monitored', job_name)
RedisToStream.monitor_job_resources(job_uuid=job_uuid)
if job_uuid in request.app.job_resources_ws_mangers:
ws_manager = request.app.job_resources_ws_mangers[job_uuid]
else:
ws_manager = SocketManager()
request.app.job_resources_ws_mangers[job_uuid] = ws_manager
def handle_job_disconnected_ws(ws):
ws_manager.remove_sockets(ws)
if not ws_manager.ws:
_logger.info('Stopping resources monitor for job %s', job_name)
RedisToStream.remove_job_resources(job_uuid=job_uuid)
request.app.job_resources_ws_mangers.pop(job_uuid, None)
_logger.info('Quitting resources socket for job %s', job_name)
ws_manager.add_socket(ws)
should_check = 0
while True:
resources = RedisToStream.get_latest_job_resources(job=job_uuid, job_name=job_name)
should_check += 1
# After trying a couple of time, we must check the status of the job
if should_check > RESOURCES_CHECK:
job.refresh_from_db()
if job.is_done:
_logger.info('removing all socket because the job `%s` is done', job_name)
ws_manager.ws = set([])
handle_job_disconnected_ws(ws)
return
else:
should_check -= CHECK_DELAY
if resources:
try:
await ws.send(resources)
except ConnectionClosed:
handle_job_disconnected_ws(ws)
return
# Just to check if connection closed
if ws._connection_lost: # pylint:disable=protected-access
handle_job_disconnected_ws(ws)
return
await asyncio.sleep(SOCKET_SLEEP)
def view_text(_, path=''):
current_directory = os.path.realpath('.')
full_path = os.path.realpath(os.path.join(current_directory, path))
if os.path.commonprefix([full_path, current_directory
]) != current_directory:
raise exceptions.Forbidden(
"Can only serve files from within the current directory")
path_parts = get_path_parts(path)
template = env.get_template('view_text.html')
contents = None
error = None
try:
contents = open(full_path, encoding="utf8", errors="replace").read()
except IOError:
error = "Could not read file."
html_content = template.render(path_parts=path_parts,
contents=contents,
error=error)
return response.html(html_content)
async def experiment_logs(request, ws, username, project_name,
experiment_sequence):
project = _get_project(username, project_name)
if not has_project_permissions(request.app.user, project, 'GET'):
exceptions.Forbidden("You don't have access to this project")
experiment = _get_validated_experiment(project, experiment_sequence)
experiment_uuid = experiment.uuid.hex
auditor.record(event_type=EXPERIMENT_LOGS_VIEWED,
instance=experiment,
actor_id=request.app.user.id)
if not RedisToStream.is_monitored_experiment_logs(
experiment_uuid=experiment_uuid):
logger.info('Experiment uuid `%s` logs is now being monitored',
experiment_uuid)
RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid)
# start consumer
if experiment_uuid in request.app.experiment_logs_consumers:
consumer = request.app.experiment_logs_consumers[experiment_uuid]
else:
logger.info('Add experiment log consumer for %s', experiment_uuid)
consumer = Consumer(
routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS,
experiment_uuid),
queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS,
experiment_uuid))
request.app.experiment_logs_consumers[experiment_uuid] = consumer
consumer.run()
# add socket manager
consumer.add_socket(ws)
should_quite = False
num_message_retries = 0
while True:
num_message_retries += 1
for message in consumer.get_messages():
num_message_retries = 0
disconnected_ws = set()
for _ws in consumer.ws:
try:
await _ws.send(message)
except ConnectionClosed:
disconnected_ws.add(_ws)
consumer.remove_sockets(disconnected_ws)
# After trying a couple of time, we must check the status of the experiment
if num_message_retries > MAX_RETRIES:
experiment.refresh_from_db()
if experiment.is_done:
logger.info(
'removing all socket because the experiment `%s` is done',
experiment_uuid)
consumer.ws = set([])
else:
num_message_retries -= CHECK_DELAY
# Just to check if connection closed
if ws._connection_lost: # pylint:disable=protected-access
logger.info('Quitting logs socket for experiment uuid %s',
experiment_uuid)
consumer.remove_sockets({
ws,
})
should_quite = True
if not consumer.ws:
logger.info('Stopping logs monitor for experiment uuid %s',
experiment_uuid)
RedisToStream.remove_experiment_logs(
experiment_uuid=experiment_uuid)
# if experiment_uuid in request.app.experiment_logs_consumers:
# consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None)
# if consumer:
# consumer.stop()
should_quite = True
if should_quite:
return
await asyncio.sleep(SOCKET_SLEEP)
async def handle_path(request, path):
# make sure we aren't serving up paths from outside of the current directory
# eg prevents curl 'http://127.0.0.1:8000/%2e%2e/' serving up the parent directory
current_directory = os.path.realpath('.')
full_path = os.path.realpath(os.path.join(current_directory, path))
if os.path.commonprefix([full_path, current_directory
]) != current_directory:
raise exceptions.Forbidden(
"Can only serve files from within the current directory")
if os.path.exists(full_path) is False:
raise exceptions.NotFound("Path does not exist")
current_user = auth.current_user(request)
is_logged_in = (current_user is not None)
if os.path.isdir(full_path) is True:
# serve an html representation of the directory
html = list_directory(full_path, path, is_logged_in)
# prevent any sort of browser caching
headers = {
'Cache-Control': 'private, max-age=0, no-cache, no-store',
}
return response.html(html, headers=headers)
else:
chunk_size = 2 * (1024**2) # 2mb
file_size = os.path.getsize(full_path)
range_object = None
range_header = request.headers.get('Range')
if range_header is not None:
range_object = types.SimpleNamespace()
range_object.start = 0
range_object.total = file_size
range_object.end = range_object.total - 1
range_ = range_header.split('=')[1]
requested_start, requested_end = range_.split('-')
if requested_start is not '':
range_object.start = int(requested_start)
if requested_end is not '':
range_object.end = int(requested_end)
range_object.end = min(range_object.end,
range_object.start + chunk_size - 1)
range_object.size = range_object.end - range_object.start + 1
headers = {
'Accept-Ranges': 'Accept-Ranges: bytes',
'Monkey-Patch-Content-Length': file_size,
}
return await response.file_stream(
full_path,
headers=headers,
chunk_size=chunk_size,
_range=range_object,
)
async def job_logs(request, ws, username, project_name, experiment_sequence,
job_sequence):
project = _get_project(username, project_name)
if not has_project_permissions(request.app.user, project, 'GET'):
exceptions.Forbidden("You don't have access to this project")
experiment = _get_validated_experiment(project, experiment_sequence)
job = _get_job(experiment, job_sequence)
job_uuid = job.uuid.hex
if not RedisToStream.is_monitored_job_logs(job_uuid=job_uuid):
logger.info(
'Job uuid `{}` logs is now being monitored'.format(job_uuid))
RedisToStream.monitor_job_logs(job_uuid=job_uuid)
# start consumer
if job_uuid in request.app.job_logs_consumers:
consumer = request.app.job_logs_consumers[job_uuid]
else:
logger.info('Add job log consumer for {}'.format(job_uuid))
consumer = Consumer(
routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS,
experiment.uuid.hex, job_uuid),
queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, job_uuid))
request.app.job_logs_consumers[job_uuid] = consumer
consumer.run()
# add socket manager
consumer.add_socket(ws)
should_quite = False
num_message_retries = 0
while True:
num_message_retries += 1
for message in consumer.get_messages():
num_message_retries = 0
disconnected_ws = set()
for _ws in consumer.ws:
try:
await _ws.send(message)
except ConnectionClosed:
disconnected_ws.add(_ws)
consumer.remove_sockets(disconnected_ws)
# After trying a couple of time, we must check the status of the experiment
if num_message_retries > MAX_RETRIES:
job.refresh_from_db()
if job.is_done:
logger.info(
'removing all socket because the job `{}` is done'.format(
job_uuid))
consumer.ws = set([])
else:
num_message_retries -= CHECK_DELAY
# Just to check if connection closed
if ws._connection_lost:
logger.info(
'Quitting logs socket for job uuid {}'.format(job_uuid))
consumer.remove_sockets({
ws,
})
should_quite = True
if len(consumer.ws) == 0:
logger.info(
'Stopping logs monitor for job uuid {}'.format(job_uuid))
RedisToStream.remove_job_logs(job_uuid=job_uuid)
# if job_uuid in request.app.job_logs_consumers:
# consumer = request.app.job_logs_consumers.pop(job_uuid, None)
# if consumer:
# consumer.stop()
should_quite = True
if should_quite:
return
await asyncio.sleep(SOCKET_SLEEP)
async def experiment_resources(request, ws, username, project_name,
experiment_sequence):
project = _get_project(username, project_name)
if not has_project_permissions(request.app.user, project, 'GET'):
exceptions.Forbidden("You don't have access to this project")
experiment = _get_validated_experiment(project, experiment_sequence)
experiment_uuid = experiment.uuid.hex
if not RedisToStream.is_monitored_experiment_resources(
experiment_uuid=experiment_uuid):
logger.info(
'Experiment resource with uuid `{}` is now being monitored'.format(
experiment_uuid))
RedisToStream.monitor_experiment_resources(
experiment_uuid=experiment_uuid)
if experiment_uuid in request.app.experiment_resources_ws_mangers:
ws_manager = request.app.experiment_resources_ws_mangers[
experiment_uuid]
else:
ws_manager = SocketManager()
request.app.experiment_resources_ws_mangers[
experiment_uuid] = ws_manager
def handle_experiment_disconnected_ws(ws):
ws_manager.remove_sockets(ws)
if len(ws_manager.ws) == 0:
logger.info('Stopping resources monitor for uuid {}'.format(
experiment_uuid))
RedisToStream.remove_experiment_resources(
experiment_uuid=experiment_uuid)
request.app.experiment_resources_ws_mangers.pop(
experiment_uuid, None)
logger.info(
'Quitting resources socket for uuid {}'.format(experiment_uuid))
jobs = []
for job in experiment.jobs.values('uuid', 'role', 'sequence'):
job['uuid'] = job['uuid'].hex
job['name'] = '{}.{}'.format(job.pop('role'), job.pop('sequence'))
jobs.append(job)
ws_manager.add_socket(ws)
should_check = 0
while True:
resources = RedisToStream.get_latest_experiment_resources(jobs)
should_check += 1
# After trying a couple of time, we must check the status of the experiment
if should_check > RESOURCES_CHECK:
experiment.refresh_from_db()
if experiment.is_done:
logger.info(
'removing all socket because the experiment `{}` is done'.
format(experiment_uuid))
ws_manager.ws = set([])
handle_experiment_disconnected_ws(ws)
return
else:
should_check -= CHECK_DELAY
if resources:
try:
await ws.send(resources)
except ConnectionClosed:
handle_experiment_disconnected_ws(ws)
return
# Just to check if connection closed
if ws._connection_lost:
handle_experiment_disconnected_ws(ws)
return
await asyncio.sleep(SOCKET_SLEEP)
