async def job_resources(request, ws, username, project_name, experiment_sequence, job_sequence): project = _get_project(username, project_name) if not has_project_permissions(request.app.user, project, 'GET'): exceptions.Forbidden("You don't have access to this project") experiment = _get_validated_experiment(project, experiment_sequence) job = _get_job(experiment, job_sequence) job_uuid = job.uuid.hex job_name = '{}.{}'.format(job.role, job.sequence) auditor.record(event_type=EXPERIMENT_JOB_RESOURCES_VIEWED, instance=job, actor_id=request.app.user.id) if not RedisToStream.is_monitored_job_resources(job_uuid=job_uuid): _logger.info('Job resources with uuid `%s` is now being monitored', job_name) RedisToStream.monitor_job_resources(job_uuid=job_uuid) if job_uuid in request.app.job_resources_ws_mangers: ws_manager = request.app.job_resources_ws_mangers[job_uuid] else: ws_manager = SocketManager() request.app.job_resources_ws_mangers[job_uuid] = ws_manager def handle_job_disconnected_ws(ws): ws_manager.remove_sockets(ws) if not ws_manager.ws: _logger.info('Stopping resources monitor for job %s', job_name) RedisToStream.remove_job_resources(job_uuid=job_uuid) request.app.job_resources_ws_mangers.pop(job_uuid, None) _logger.info('Quitting resources socket for job %s', job_name) ws_manager.add_socket(ws) should_check = 0 while True: resources = RedisToStream.get_latest_job_resources(job=job_uuid, job_name=job_name) should_check += 1 # After trying a couple of time, we must check the status of the job if should_check > RESOURCES_CHECK: job.refresh_from_db() if job.is_done: _logger.info('removing all socket because the job `%s` is done', job_name) ws_manager.ws = set([]) handle_job_disconnected_ws(ws) return else: should_check -= CHECK_DELAY if resources: try: await ws.send(resources) except ConnectionClosed: handle_job_disconnected_ws(ws) return # Just to check if connection closed if ws._connection_lost: # pylint:disable=protected-access handle_job_disconnected_ws(ws) return await asyncio.sleep(SOCKET_SLEEP)
def view_text(_, path=''): current_directory = os.path.realpath('.') full_path = os.path.realpath(os.path.join(current_directory, path)) if os.path.commonprefix([full_path, current_directory ]) != current_directory: raise exceptions.Forbidden( "Can only serve files from within the current directory") path_parts = get_path_parts(path) template = env.get_template('view_text.html') contents = None error = None try: contents = open(full_path, encoding="utf8", errors="replace").read() except IOError: error = "Could not read file." html_content = template.render(path_parts=path_parts, contents=contents, error=error) return response.html(html_content)
async def experiment_logs(request, ws, username, project_name, experiment_sequence): project = _get_project(username, project_name) if not has_project_permissions(request.app.user, project, 'GET'): exceptions.Forbidden("You don't have access to this project") experiment = _get_validated_experiment(project, experiment_sequence) experiment_uuid = experiment.uuid.hex auditor.record(event_type=EXPERIMENT_LOGS_VIEWED, instance=experiment, actor_id=request.app.user.id) if not RedisToStream.is_monitored_experiment_logs( experiment_uuid=experiment_uuid): logger.info('Experiment uuid `%s` logs is now being monitored', experiment_uuid) RedisToStream.monitor_experiment_logs(experiment_uuid=experiment_uuid) # start consumer if experiment_uuid in request.app.experiment_logs_consumers: consumer = request.app.experiment_logs_consumers[experiment_uuid] else: logger.info('Add experiment log consumer for %s', experiment_uuid) consumer = Consumer( routing_key='{}.{}.*'.format(RoutingKeys.LOGS_SIDECARS, experiment_uuid), queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, experiment_uuid)) request.app.experiment_logs_consumers[experiment_uuid] = consumer consumer.run() # add socket manager consumer.add_socket(ws) should_quite = False num_message_retries = 0 while True: num_message_retries += 1 for message in consumer.get_messages(): num_message_retries = 0 disconnected_ws = set() for _ws in consumer.ws: try: await _ws.send(message) except ConnectionClosed: disconnected_ws.add(_ws) consumer.remove_sockets(disconnected_ws) # After trying a couple of time, we must check the status of the experiment if num_message_retries > MAX_RETRIES: experiment.refresh_from_db() if experiment.is_done: logger.info( 'removing all socket because the experiment `%s` is done', experiment_uuid) consumer.ws = set([]) else: num_message_retries -= CHECK_DELAY # Just to check if connection closed if ws._connection_lost: # pylint:disable=protected-access logger.info('Quitting logs socket for experiment uuid %s', experiment_uuid) consumer.remove_sockets({ ws, }) should_quite = True if not consumer.ws: logger.info('Stopping logs monitor for experiment uuid %s', experiment_uuid) RedisToStream.remove_experiment_logs( experiment_uuid=experiment_uuid) # if experiment_uuid in request.app.experiment_logs_consumers: # consumer = request.app.experiment_logs_consumers.pop(experiment_uuid, None) # if consumer: # consumer.stop() should_quite = True if should_quite: return await asyncio.sleep(SOCKET_SLEEP)
async def handle_path(request, path): # make sure we aren't serving up paths from outside of the current directory # eg prevents curl 'http://127.0.0.1:8000/%2e%2e/' serving up the parent directory current_directory = os.path.realpath('.') full_path = os.path.realpath(os.path.join(current_directory, path)) if os.path.commonprefix([full_path, current_directory ]) != current_directory: raise exceptions.Forbidden( "Can only serve files from within the current directory") if os.path.exists(full_path) is False: raise exceptions.NotFound("Path does not exist") current_user = auth.current_user(request) is_logged_in = (current_user is not None) if os.path.isdir(full_path) is True: # serve an html representation of the directory html = list_directory(full_path, path, is_logged_in) # prevent any sort of browser caching headers = { 'Cache-Control': 'private, max-age=0, no-cache, no-store', } return response.html(html, headers=headers) else: chunk_size = 2 * (1024**2) # 2mb file_size = os.path.getsize(full_path) range_object = None range_header = request.headers.get('Range') if range_header is not None: range_object = types.SimpleNamespace() range_object.start = 0 range_object.total = file_size range_object.end = range_object.total - 1 range_ = range_header.split('=')[1] requested_start, requested_end = range_.split('-') if requested_start is not '': range_object.start = int(requested_start) if requested_end is not '': range_object.end = int(requested_end) range_object.end = min(range_object.end, range_object.start + chunk_size - 1) range_object.size = range_object.end - range_object.start + 1 headers = { 'Accept-Ranges': 'Accept-Ranges: bytes', 'Monkey-Patch-Content-Length': file_size, } return await response.file_stream( full_path, headers=headers, chunk_size=chunk_size, _range=range_object, )
async def job_logs(request, ws, username, project_name, experiment_sequence, job_sequence): project = _get_project(username, project_name) if not has_project_permissions(request.app.user, project, 'GET'): exceptions.Forbidden("You don't have access to this project") experiment = _get_validated_experiment(project, experiment_sequence) job = _get_job(experiment, job_sequence) job_uuid = job.uuid.hex if not RedisToStream.is_monitored_job_logs(job_uuid=job_uuid): logger.info( 'Job uuid `{}` logs is now being monitored'.format(job_uuid)) RedisToStream.monitor_job_logs(job_uuid=job_uuid) # start consumer if job_uuid in request.app.job_logs_consumers: consumer = request.app.job_logs_consumers[job_uuid] else: logger.info('Add job log consumer for {}'.format(job_uuid)) consumer = Consumer( routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS, experiment.uuid.hex, job_uuid), queue='{}.{}'.format(CeleryQueues.STREAM_LOGS_SIDECARS, job_uuid)) request.app.job_logs_consumers[job_uuid] = consumer consumer.run() # add socket manager consumer.add_socket(ws) should_quite = False num_message_retries = 0 while True: num_message_retries += 1 for message in consumer.get_messages(): num_message_retries = 0 disconnected_ws = set() for _ws in consumer.ws: try: await _ws.send(message) except ConnectionClosed: disconnected_ws.add(_ws) consumer.remove_sockets(disconnected_ws) # After trying a couple of time, we must check the status of the experiment if num_message_retries > MAX_RETRIES: job.refresh_from_db() if job.is_done: logger.info( 'removing all socket because the job `{}` is done'.format( job_uuid)) consumer.ws = set([]) else: num_message_retries -= CHECK_DELAY # Just to check if connection closed if ws._connection_lost: logger.info( 'Quitting logs socket for job uuid {}'.format(job_uuid)) consumer.remove_sockets({ ws, }) should_quite = True if len(consumer.ws) == 0: logger.info( 'Stopping logs monitor for job uuid {}'.format(job_uuid)) RedisToStream.remove_job_logs(job_uuid=job_uuid) # if job_uuid in request.app.job_logs_consumers: # consumer = request.app.job_logs_consumers.pop(job_uuid, None) # if consumer: # consumer.stop() should_quite = True if should_quite: return await asyncio.sleep(SOCKET_SLEEP)
async def experiment_resources(request, ws, username, project_name, experiment_sequence): project = _get_project(username, project_name) if not has_project_permissions(request.app.user, project, 'GET'): exceptions.Forbidden("You don't have access to this project") experiment = _get_validated_experiment(project, experiment_sequence) experiment_uuid = experiment.uuid.hex if not RedisToStream.is_monitored_experiment_resources( experiment_uuid=experiment_uuid): logger.info( 'Experiment resource with uuid `{}` is now being monitored'.format( experiment_uuid)) RedisToStream.monitor_experiment_resources( experiment_uuid=experiment_uuid) if experiment_uuid in request.app.experiment_resources_ws_mangers: ws_manager = request.app.experiment_resources_ws_mangers[ experiment_uuid] else: ws_manager = SocketManager() request.app.experiment_resources_ws_mangers[ experiment_uuid] = ws_manager def handle_experiment_disconnected_ws(ws): ws_manager.remove_sockets(ws) if len(ws_manager.ws) == 0: logger.info('Stopping resources monitor for uuid {}'.format( experiment_uuid)) RedisToStream.remove_experiment_resources( experiment_uuid=experiment_uuid) request.app.experiment_resources_ws_mangers.pop( experiment_uuid, None) logger.info( 'Quitting resources socket for uuid {}'.format(experiment_uuid)) jobs = [] for job in experiment.jobs.values('uuid', 'role', 'sequence'): job['uuid'] = job['uuid'].hex job['name'] = '{}.{}'.format(job.pop('role'), job.pop('sequence')) jobs.append(job) ws_manager.add_socket(ws) should_check = 0 while True: resources = RedisToStream.get_latest_experiment_resources(jobs) should_check += 1 # After trying a couple of time, we must check the status of the experiment if should_check > RESOURCES_CHECK: experiment.refresh_from_db() if experiment.is_done: logger.info( 'removing all socket because the experiment `{}` is done'. format(experiment_uuid)) ws_manager.ws = set([]) handle_experiment_disconnected_ws(ws) return else: should_check -= CHECK_DELAY if resources: try: await ws.send(resources) except ConnectionClosed: handle_experiment_disconnected_ws(ws) return # Just to check if connection closed if ws._connection_lost: handle_experiment_disconnected_ws(ws) return await asyncio.sleep(SOCKET_SLEEP)