def get_jobs_for_runner( public_runner_id: uuid.UUID ) -> cg_json.JSONResponse[t.List[t.Mapping[str, str]]]: """Get jobs for a runner""" runner = db.session.query(models.Runner).filter( models.Runner.ipaddr == request.remote_addr, models.Runner.public_id == public_runner_id, t.cast(DbColumn[models.RunnerState], models.Runner.state).in_( models.RunnerState.get_before_running_states()), ).one_or_none() if runner is None: raise NotFoundException if isinstance(runner, models.AWSRunner): runner_pass = request.headers.get('CG-Broker-Runner-Pass', '') if not runner.is_pass_valid(runner_pass): logger.warning('Got wrong password', found_password=runner_pass) raise NotFoundException urls = set(url for url, in db.session.query( t.cast(DbColumn[str], models.Job.cg_url), ).filter(~t.cast( DbColumn[models.JobState], models.Job.state, ).in_(models.JobState.get_finished_states()))) return cg_json.jsonify([{'url': url} for url in urls])
def register_job() -> cg_json.JSONResponse: """Register a new job. If needed a runner will be started for this job. """ remote_job_id = g.data['job_id'] cg_url = g.cg_instance_url job = None if request.method == 'PUT': job = db.session.query(models.Job).filter_by( remote_id=remote_job_id, cg_url=cg_url, ).with_for_update().one_or_none() will_create = job is None if will_create and g.data.get('error_on_create', False): raise NotFoundException if job is None: job = models.Job( remote_id=remote_job_id, cg_url=cg_url, ) db.session.add(job) job.update_metadata(g.data.get('metadata', {})) if 'wanted_runners' in g.data or will_create: job.wanted_runners = g.data.get('wanted_runners', 1) active_runners = models.Runner.get_all_active_runners().filter_by( job_id=job.id ).with_for_update().all() too_many = len(active_runners) - job.wanted_runners logger.info( 'Job was updated', wanted_runners=job.wanted_runners, amount_active=len(active_runners), too_many=too_many, metadata=job.job_metadata, ) before_assigned = set(models.RunnerState.get_before_assigned_states()) for runner in active_runners: if too_many <= 0: break if runner.state in before_assigned: runner.make_unassigned() too_many -= 1 db.session.flush() job_id = job.id callback_after_this_request( lambda: tasks.maybe_start_runners_for_job.delay(job_id) ) db.session.commit() return cg_json.jsonify(job)
def about() -> cg_json.JSONResponse[t.Mapping[str, object]]: """Get some information about the state of this broker. When given a valid ``health`` get parameter this will also return some health information. """ if request.args.get('health', object()) == app.config['HEALTH_KEY']: now = DatetimeWithTimezone.utcnow() slow_created_date = now - timedelta(minutes=app.config['OLD_JOB_AGE']) not_started_created_date = now - timedelta( minutes=app.config['SLOW_STARTING_AGE'] ) not_started_task_date = now - timedelta( minutes=app.config['SLOW_STARTING_TASK_AGE'] ) slow_task_date = now - timedelta(minutes=app.config['SLOW_TASK_AGE']) def get_count(*cols: DbColumn[bool]) -> int: return db.session.query(models.Job).filter( models.Job.state.notin_(models.JobState.get_finished_states()), *cols, ).count() slow_jobs = get_count(models.Job.created_at < slow_created_date) not_starting_jobs = get_count( models.Job.created_at < not_started_created_date, models.Job.state == models.JobState.waiting_for_runner, ) def as_dt(col: IndexedJSONColumn) -> DbColumn[DatetimeWithTimezone]: return col.as_string().cast(TIMESTAMP(timezone=True)) not_started_task = models.Job.job_metadata['results']['not_started'] jobs_not_starting_tasks = get_count( as_dt(not_started_task) < not_started_task_date ) slow_task = models.Job.job_metadata['results']['running'] jobs_with_slow_tasks = get_count(as_dt(slow_task) < slow_task_date) health = { 'not_starting_jobs': not_starting_jobs, 'slow_jobs': slow_jobs, 'jobs_with_not_starting_tasks': jobs_not_starting_tasks, 'jobs_with_slow_tasks': jobs_with_slow_tasks, } else: health = {} return cg_json.jsonify( { 'health': health, 'version': app.config.get('CUR_COMMIT', 'unknown'), }, status_code=500 if any(health.values()) else 200, )
def handle_404(_: object) -> JSONResponse[APIException]: # pylint: disable=unused-variable; #pragma: no cover from . import models # pylint: disable=import-outside-toplevel models.db.session.expire_all() logger.warning('A unknown route was requested') api_exp = APIException('The request route was not found', f'The route "{request.path}" does not exist', APICodes.ROUTE_NOT_FOUND, 404) return jsonify(api_exp, status_code=404)
def handle_404(_: object) -> JSONResponse[APIException]: # pylint: disable=unused-variable; #pragma: no cover logger.warning('A unknown route was requested') api_exp = APIException('The request route was not found', f'The route "{request.path}" does not exist', APICodes.ROUTE_NOT_FOUND, 404) psef.models.db.session.rollback() return jsonify(api_exp, status_code=404)
def handle_404(_: object) -> Response: # pylint: disable=unused-variable; #pragma: no cover from . import models models.db.session.expire_all() api_exp = APIException('The request route was not found', f'The route "{request.path}" does not exist', APICodes.ROUTE_NOT_FOUND, 404) response = t.cast(t.Any, jsonify(api_exp)) logger.warning('A unknown route was requested') response.status_code = 404 return response
def __on_unknown_error(_: APIException) -> Response: logger.warning('Unknown exception occurred', exc_info=True) res = t.cast( t.Any, jsonify({ 'error': 'Something unknown went wrong! (request_id: {})'.format( g.request_id) })) res.status_code = 500 return res
def copy_auto_test(auto_test_id: int) -> JSONResponse[models.AutoTest]: """Copy the given AutoTest configuration. .. :quickref: AutoTest; Copy an AutoTest config to another assignment. :param auto_test_id: The id of the AutoTest config which should be copied. :returns: The copied AutoTest configuration. """ data = rqa.FixedMapping( rqa.RequiredArgument( 'assignment_id', rqa.SimpleValue.int, """ The id of the assignment into which you want to copy this AutoTest. """, )).from_flask() test = get_or_404(models.AutoTest, auto_test_id, also_error=lambda at: not at.assignment.is_visible) auth.AutoTestPermissions(test).ensure_may_see() for fixture in test.fixtures: auth.AutoTestFixturePermissions(fixture).ensure_may_see() for suite in test.all_suites: for step in suite.steps: auth.ensure_can_view_autotest_step_details(step) assignment = filter_single_or_404( models.Assignment, models.Assignment.id == data.assignment_id, with_for_update=True) auth.ensure_permission(CPerm.can_edit_autotest, assignment.course_id) if assignment.auto_test is not None: raise APIException( 'The given assignment already has an AutoTest', f'The assignment "{assignment.id}" already has an auto test', APICodes.INVALID_STATE, 409) assignment.rubric_rows = [] mapping = {} for old_row in test.assignment.rubric_rows: new_row = old_row.copy() mapping[old_row] = new_row assignment.rubric_rows.append(new_row) db.session.flush() with app.file_storage.putter() as putter: assignment.auto_test = test.copy(mapping, putter) db.session.flush() db.session.commit() return jsonify(assignment.auto_test)
def __handle_error(_: RateLimitExceeded) -> Response: # pylint: disable=unused-variable res = t.cast( Response, jsonify( errors.APIException( 'Rate limit exceeded, slow down!', 'Rate limit is exceeded', errors.APICodes.RATE_LIMIT_EXCEEDED, 429, ))) res.status_code = 429 return res
def second_phase_lti_launch( ) -> helpers.JSONResponse[t.Union[_LTILaunchResult, t.Dict[str, t.Any]]]: """Do the second part of an LTI launch. :>json string blob_id: The id of the blob which you got from the lti launch redirect. :returns: A _LTILaunch instance. :raises APIException: If the given Jwt token is not valid. (INVALID_PARAM) """ with helpers.get_from_map_transaction( helpers.get_json_dict_from_request()) as [get, _]: blob_id = get('blob_id', str) res = _get_second_phase_lti_launch_data(blob_id) db.session.commit() if helpers.extended_requested(): return jsonify(res) return jsonify(t.cast(dict, res['data']))
def __handle_404( _: object) -> t.Union[str, JSONResponse[dict]]: # pragma: no cover if request.path.startswith('/api/'): return jsonify( { 'error': 'Something unknown went wrong! (request_id: {})'.format( g.request_id) }, status_code=404, ) return flask.render_template('404.j2')
def get_lti1_3_config(lti_provider_id: str) -> helpers.JSONResponse[object]: """Get the LTI 1.3 config in such a way that it can be used by the LMS connected to this provider. :param lti_provider_id: The id of the provider you want to get the configuration for. :returns: An opaque object that is useful only for the LMS. """ lti_provider = helpers.filter_single_or_404( models.LTI1p3Provider, models.LTI1p3Provider.id == lti_provider_id) return jsonify(lti_provider.get_json_config())
def register_job() -> cg_json.JSONResponse: """Register a new job. If needed a runner will be started for this job. """ remote_job_id = g.data['job_id'] cg_url = request.headers['CG-Broker-Instance'] job = None if request.method == 'PUT': job = db.session.query(models.Job).filter_by( remote_id=remote_job_id, cg_url=cg_url, ).one_or_none() if job is None: job = models.Job( remote_id=remote_job_id, cg_url=cg_url, ) db.session.add(job) job.update_metadata(g.data.get('metadata', {})) job.wanted_runners = g.data.get('wanted_runners', 1) active_runners = models.Runner.get_all_active_runners().filter_by( job_id=job.id).with_for_update().all() too_many = len(active_runners) - job.wanted_runners logger.info( 'Job was updated', wanted_runners=job.wanted_runners, amount_active=len(active_runners), too_many=too_many, metadata=job.job_metadata, ) for runner in active_runners: if too_many <= 0: break if runner.state in models.RunnerState.get_before_running_states(): runner.make_unassigned() too_many -= 1 db.session.commit() job_id = job.id callback_after_this_request( lambda: tasks.maybe_start_runners_for_job.delay(job_id)) assert job.id is not None return cg_json.jsonify(job)
def update_lti1p3_provider( lti_provider_id: str) -> helpers.JSONResponse[models.LTI1p3Provider]: """Update the given LTI 1.3 provider. .. :quickref: LTI; Update the information of an LTI 1.3 provider. This route is part of the public API. :param lti_provider_id: The id of the provider you want to update. :<json str client_id: The new client id. :<json str auth_token_url: The new authentication token url. :<json str auth_login_url: The new authentication login url. :<json str key_set_url: The new key set url. :<json str auth_audience: The new OAuth2 audience. :<json bool finalize: Should this provider be finalized. All input JSON is optional, when not provided that attribute will not be updated. :returns: The updated LTI 1.3 provider. """ lti_provider = helpers.filter_single_or_404( models.LTI1p3Provider, models.LTI1p3Provider.id == lti_provider_id) secret = flask.request.args.get('secret') auth.LTI1p3ProviderPermissions(lti_provider, secret=secret).ensure_may_edit() with helpers.get_from_request_transaction() as [_, opt_get]: iss = opt_get('iss', str, None) client_id = opt_get('client_id', str, None) auth_token_url = opt_get('auth_token_url', str, None) auth_login_url = opt_get('auth_login_url', str, None) key_set_url = opt_get('key_set_url', str, None) auth_audience = opt_get('auth_audience', str, None) finalize = opt_get('finalize', bool, None) lti_provider.update_registration( iss=iss, client_id=client_id, auth_token_url=auth_token_url, auth_login_url=auth_login_url, key_set_url=key_set_url, auth_audience=auth_audience, finalize=finalize, ) db.session.commit() return jsonify(lti_provider)
def __on_unknown_error(_: Exception) -> JSONResponse[t.Dict[str, str]]: logger.error( 'Unknown exception occurred', exc_info=True, report_to_sentry=True, ) return jsonify( { 'error': 'Something unknown went wrong! (request_id: {})'.format( g.request_id) }, status_code=500, )
def list_lti1p3_provider( ) -> helpers.JSONResponse[t.List[models.LTI1p3Provider]]: """List all known LTI 1.3 providers for this instance. .. :quickref: LTI; List all known LTI 1.3 providers. This route is part of the public API. :returns: A list of all known LTI 1.3 providers. """ providers = [ prov for prov in models.LTI1p3Provider.query.order_by( models.LTI1p3Provider.created_at.asc(), ) if auth.LTI1p3ProviderPermissions(prov).ensure_may_see.as_bool() ] return jsonify(providers)
def __handle_unknown_error(_: Exception) -> Response: # pylint: disable=unused-variable; #pragma: no cover """Handle an unhandled error. This function should never really be called, as it means our code contains a bug. """ from . import models models.db.session.expire_all() api_exp = APIException(f'Something went wrong (id: {g.request_id})', ('The reason for this is unknown, ' 'please contact the system administrator'), APICodes.UNKOWN_ERROR, 500) response = t.cast(t.Any, jsonify(api_exp)) response.status_code = 500 logger.error('Unknown exception occurred', exc_info=True) return response
def get_lti_provider_jwks( lti_provider_id: str ) -> helpers.JSONResponse[t.Mapping[str, t.List[t.Mapping[str, str]]]]: """Get the JWKS of a given provider. .. :quickref: LTI; Get the public key of a provider in JWKS format. The ``/api/v1/lti1.3/providers/<lti_provider_id>/jwks`` route is part of the public API. The ``/api/v1/lti1.3/jwks/<lti_provider_id>`` is not and **will** be removed in a future version. :param lti_provider_id: The id of the provider from which you want to get the JWKS. """ lti_provider = helpers.filter_single_or_404( models.LTI1p3Provider, models.LTI1p3Provider.id == lti_provider_id) return jsonify({'keys': [lti_provider.get_public_jwk()]})
def get_lti1p3_provider( lti_provider_id: str) -> helpers.JSONResponse[models.LTI1p3Provider]: """Get a LTI 1.3 provider. .. :quickref: LTI; Get a LTI 1.3 provider by id. This route is part of the public API. :param lti_provider_id: The id of the provider you want to get. :returns: The requested LTI 1.3 provider. """ lti_provider = helpers.filter_single_or_404( models.LTI1p3Provider, models.LTI1p3Provider.id == lti_provider_id) secret = flask.request.args.get('secret') auth.LTIProviderPermissions(lti_provider, secret=secret).ensure_may_see() return jsonify(lti_provider)
def get_auto_test_result_proxy( auto_test_id: int, run_id: int, result_id: int, suite_id: int, ) -> JSONResponse[models.Proxy]: """Create a proxy to view the files of the given AT result through. .. :quickref: AutoTest; Create a proxy to view the files a result. This allows you to view files of an AutoTest result (within a suite) without authentication for a limited time. :param auto_test_id: The id of the AutoTest in which the result is located. :param run_id: The id of run in which the result is located. :param result_id: The id of the result from which you want to get the files. :param suite_id: The suite from which you want to proxy the output files. :<json bool allow_remote_resources: Allow the proxy to load remote resources. :<json bool allow_remote_scripts: Allow the proxy to load remote scripts, and allow to usage of 'eval'. :returns: The created proxy. """ with get_from_map_transaction(get_json_dict_from_request()) as [get, _]: allow_remote_resources = get('allow_remote_resources', bool) allow_remote_scripts = get('allow_remote_scripts', bool) result = _get_result_by_ids(auto_test_id, run_id, result_id) auth.AutoTestResultPermissions(result).ensure_may_see_output_files() base_file = filter_single_or_404( models.AutoTestOutputFile, models.AutoTestOutputFile.parent_id.is_(None), models.AutoTestOutputFile.auto_test_suite_id == suite_id, models.AutoTestOutputFile.result == result, ) proxy = models.Proxy( base_at_result_file=base_file, allow_remote_resources=allow_remote_resources, allow_remote_scripts=allow_remote_scripts, ) db.session.add(proxy) db.session.commit() return jsonify(proxy)
def update_auto_test_set( auto_test_id: int, auto_test_set_id: int) -> JSONResponse[models.AutoTestSet]: """Update the given :class:`.models.AutoTestSet`. .. :quickref: AutoTest; Update a single AutoTest set. :>json stop_points: The minimum amount of points a student should have after this set to continue testing. :param auto_test_id: The id of the :class:`.models.AutoTest` of the set that should be updated. :param auto_test_set_id: The id of the :class:`.models.AutoTestSet` that should be updated. :returns: The updated set. """ data = rqa.FixedMapping( rqa.OptionalArgument( 'stop_points', rqa.SimpleValue.float, """ The minimum percentage a student should have achieved before the next tests will be run. """)).from_flask() auto_test_set = _get_at_set_by_ids(auto_test_id, auto_test_set_id) auth.AutoTestPermissions(auto_test_set.auto_test).ensure_may_edit() auto_test_set.auto_test.ensure_no_runs() if data.stop_points.is_just: stop_points = data.stop_points.value if stop_points < 0: raise APIException( 'You cannot set stop points to lower than 0', f"The given value for stop points ({stop_points}) isn't valid", APICodes.INVALID_PARAM, 400) elif stop_points > 1: raise APIException( 'You cannot set stop points to higher than 1', f"The given value for stop points ({stop_points}) isn't valid", APICodes.INVALID_PARAM, 400) auto_test_set.stop_points = stop_points db.session.commit() return jsonify(auto_test_set)
def __handle_unknown_error( _: Exception) -> JSONResponse[APIException]: # pragma: no cover """Handle an unhandled error. This function should never really be called, as it means our code contains a bug. """ from . import models # pylint: disable=import-outside-toplevel models.db.session.expire_all() logger.error('Unknown exception occurred', exc_info=True, report_to_sentry=True) api_exp = APIException(f'Something went wrong (id: {g.request_id})', ('The reason for this is unknown, ' 'please contact the system administrator'), APICodes.UNKOWN_ERROR, 500) return jsonify(api_exp, status_code=500)
def handle_api_error(error: APIException) -> Response: """Handle an :class:`APIException` by converting it to a :class:`flask.Response`. :param APIException error: The error that occurred :returns: A response with the JSON serialized error as content. :rtype: flask.Response """ response = jsonify(error) response.status_code = error.status_code logger.warning( 'APIException occurred', api_exception=error.__to_json__(), exc_info=True, ) psef.models.db.session.rollback() return response
def handle_api_error(error: APIException) -> Response: # pylint: disable=unused-variable """Handle an :class:`APIException` by converting it to a :class:`flask.Response`. :param APIException error: The error that occurred :returns: A response with the JSON serialized error as content. :rtype: flask.Response """ from . import models models.db.session.expire_all() response = t.cast(t.Any, jsonify(error)) response.status_code = error.status_code logger.warning( 'APIException occurred', api_exception=error.__to_json__(), exc_info=True, ) return response
def create_lti_provider() -> helpers.JSONResponse[models.LTIProviderBase]: """Create a new LTI 1.1 or 1.3 provider. .. :quickref: LTI; Create a new LTI 1.1 or 1.3 provider. This route is part of the public API. :<json lti_version: The LTI version of the new provider, defaults to ``lti1.3``. Allowed values: ``lti1.1``, ``lti1.3``. :<json str lms: The LMS of the new provider, this should be a known LMS. :<json str indented_use: The intended use of the provider. Like which organization will be using the provider, this can be any string, but cannot be empty. :<json str iss: The iss of the new provider, only required when ``lti_version`` is not given or is ``lti1.3``. When required this cannot be empty. :returns: The just created provider. """ with helpers.get_from_request_transaction() as [get, opt_get]: lti_version = opt_get('lti_version', str, 'lti1.3') intended_use = get('intended_use', str) if not intended_use: raise exceptions.APIException( 'The "intended_use" must be non empty', f'The intended_use={intended_use} was empty', exceptions.APICodes.INVALID_PARAM, 400) lti_provider: models.LTIProviderBase if lti_version == 'lti1.3': lti_provider = _create_lti1p3_provider(intended_use) elif lti_version == 'lti1.1': lti_provider = _create_lti1p1_provider(intended_use) else: assert False auth.LTIProviderPermissions(lti_provider).ensure_may_add() db.session.add(lti_provider) db.session.commit() return jsonify(lti_provider)
def update_auto_test(auto_test_id: int) -> JSONResponse[models.AutoTest]: """Update the settings of an AutoTest configuration. .. :quickref: AutoTest; Change the settings/upload fixtures to an AutoTest. :>json old_fixtures: The old fixtures you want to keep in this AutoTest (OPTIONAL). Not providing this option keeps all old fixtures. :>json setup_script: The setup script of this AutoTest (OPTIONAL). :>json run_setup_script: The run setup script of this AutoTest (OPTIONAL). :>json has_new_fixtures: If set to true you should provide one or more new fixtures in the ``POST`` (OPTIONAL). :>json grade_calculation: The way the rubric grade should be calculated from the amount of achieved points (OPTIONAL). :param auto_test_id: The id of the AutoTest you want to update. :returns: The updated AutoTest. """ data, request_files = rqa.MultipartUpload( _ATUpdateMap, file_key='fixture', multiple=True, ).from_flask() auto_test = get_or_404(models.AutoTest, auto_test_id, also_error=lambda at: not at.assignment.is_visible) auth.AutoTestPermissions(auto_test).ensure_may_edit() auto_test.ensure_no_runs() _update_auto_test( auto_test, request_files, data.fixtures, data.setup_script, data.run_setup_script, data.has_new_fixtures, data.grade_calculation, data.results_always_visible, data.prefer_teacher_revision, ) db.session.commit() return jsonify(auto_test)
def get_jobs_for_runner(public_runner_id: uuid.UUID ) -> cg_json.JSONResponse[t.List[t.Mapping[str, str]]]: """Get jobs for a runner""" runner = db.session.query(models.Runner).filter( models.Runner.ipaddr == request.remote_addr, models.Runner.public_id == public_runner_id, models.Runner.state.in_( models.RunnerState.get_before_running_states() ), ).with_for_update().one_or_none() if runner is None: raise NotFoundException runner.verify_password() # If an assigned runners comes asking for work again we simply assume that # the application backend was not successful in using the runner. So we # simply make it unassigned again. if runner.state.is_assigned and ( runner.job is None or not runner.job.needs_more_runners ): runner.make_unassigned() urls: t.Iterable[str] if runner.state in models.RunnerState.get_before_assigned_states(): urls = set( url for url, in db.session.query(models.Job.cg_url).filter( models.Job.state.notin_( models.JobState.get_finished_states(), ), models.Job.needs_more_runners, ) ) # If assigned make sure that url is first in the list so that the # runner tries that first. if runner.job is not None: best_url = runner.job.cg_url urls = sorted(urls, key=lambda url: url == best_url, reverse=True) else: urls = [] if runner.job is None else [runner.job.cg_url] return cg_json.jsonify([{'url': url} for url in urls])
def get_lti_provider( lti_provider_id: str) -> helpers.JSONResponse[models.LTIProviderBase]: """Get a LTI provider. .. :quickref: LTI; Get a LTI 1.1 or 1.3 provider by id. This route is part of the public API. :param lti_provider_id: The id of the provider you want to get. :returns: The requested LTI 1.1 or 1.3 provider. """ lti_provider: models.LTIProviderBase = helpers.filter_single_or_404( # We may only provide concrete classes when a type is expected. models.LTIProviderBase, # type: ignore[misc] models.LTIProviderBase.id == lti_provider_id, ) secret = flask.request.args.get('secret') auth.LTIProviderPermissions(lti_provider, secret=secret).ensure_may_see() return jsonify(lti_provider)
def finalize_lti1p1_provider( lti_provider_id: str) -> helpers.JSONResponse[models.LTI1p1Provider]: """Finalize the given LTI 1.1 provider. .. :quickref: LTI; Finalize a LTI 1.1 provider. This route is part of the public api. :param lti_provider_id: The id of the provider you want to finalize. """ lti_provider = helpers.filter_single_or_404( models.LTI1p1Provider, models.LTI1p1Provider.id == lti_provider_id, ) secret = flask.request.args.get('secret') auth.LTIProviderPermissions(lti_provider, secret=secret).ensure_may_edit() lti_provider.finalize() db.session.commit() return jsonify(lti_provider)
def create_auto_test_set( auto_test_id: int) -> JSONResponse[models.AutoTestSet]: """Create a new set within an AutoTest .. :quickref: AutoTest; Create a set within an AutoTest. :param auto_test_id: The id of the AutoTest wherein you want to create a set. :returns: The newly created set. """ auto_test = get_or_404(models.AutoTest, auto_test_id, also_error=lambda at: not at.assignment.is_visible) auth.AutoTestPermissions(auto_test).ensure_may_edit() auto_test.ensure_no_runs() auto_test.sets.append(models.AutoTestSet()) db.session.commit() return jsonify(auto_test.sets[-1])