def grade_submission(env, **kwargs): """Grades submision on a `Job` layer. This `Handler` aggregetes score from graded groups and gets submission status from tests results. Used ``environ`` keys: * ``group_results`` * ``test_results`` * ``score_aggregator`` Produced ``environ`` keys: * ``status`` * ``score`` """ # TODO: let score_aggregator handle compilation errors if env.get("compilation_result", "OK") != "OK": env["score"] = None env["status"] = "CE" return env fun = get_object_by_dotted_name(env.get("score_aggregator") or DEFAULT_SCORE_AGGREGATOR) group_results = dict(filter(lambda (name, res): res["kind"] == "NORMAL", env["group_results"].iteritems())) score, status = fun(group_results) assert isinstance(score, (types.NoneType, ScoreValue)) env["score"] = score and score.serialize() env["status"] = status return env
def create_env(self, request, contest, form, path, package, existing_problem=None, original_filename=None): """Creates an environment which will be later passed to :func:`~oioioi.problems.unpackmgr.unpackmgr_job`. """ backend_name = self.choose_backend(path, original_filename) backend = get_object_by_dotted_name(backend_name)() if package.problem: package.problem_name = package.problem.short_name else: package.problem_name = backend.get_short_name(path, original_filename) package.save() env = {} env['post_upload_handlers'] = \ ['oioioi.problems.handlers.update_problem_instance'] env['backend_name'] = backend_name env['package_id'] = package.id env['round_id'] = form.cleaned_data.get('round_id', None) if contest: env['contest_id'] = contest.id env['author'] = request.user.username if existing_problem: env['is_reupload'] = True else: env['is_reupload'] = False return env
def grade_submission(env, kind='NORMAL', **kwargs): """Grades submision on a `Job` layer. This `Handler` aggregetes score from graded groups and gets submission status from tests results. Used ``environ`` keys: * ``group_results`` * ``test_results`` * ``score_aggregator`` Produced ``environ`` keys: * ``status`` * ``score`` """ # TODO: let score_aggregator handle compilation errors if env.get('compilation_result', 'OK') != 'OK': env['score'] = None env['status'] = 'CE' return env fun = get_object_by_dotted_name(env.get('score_aggregator') or DEFAULT_SCORE_AGGREGATOR) group_results = dict(filter(lambda (name, res): res['kind'] == kind, env['group_results'].iteritems())) score, status = fun(group_results) assert isinstance(score, (types.NoneType, ScoreValue)) env['score'] = score and score.serialize() env['status'] = status return env
def get_client(): """Constructs a Filetracker client. Needs a ``FILETRACKER_CLIENT_FACTORY`` entry in ``settings.py``, which should contain a :term:`dotted name` of a function which returns a :class:`filetracker.Client` instance. A good candidate is :func:`~oioioi.filetracker.client.media_root_factory`. The constructed client is cached. """ factory = settings.FILETRACKER_CLIENT_FACTORY if isinstance(factory, basestring): factory = get_object_by_dotted_name(factory) if not callable(factory): raise ImproperlyConfigured('The FILETRACKER_CLIENT_FACTORY setting ' 'refers to non-callable: %r' % (factory,)) client = factory() if not isinstance(client, filetracker.Client): raise ImproperlyConfigured('The factory pointed by ' 'FILETRACKER_CLIENT_FACTORY returned non-filetracker.Client: ' '%r' % (client,)) # Needed for oioioi.sioworkers.backends.LocalBackend so that both Django # and sioworkers use the same Filetracker client import sio.workers.ft sio.workers.ft.set_instance(client) return client
def grade_tests(env, **kwargs): """Grades tests using a scoring function. The ``env['test_scorer']``, which is used by this ``Handler``, should be a path to a function which gets test definition (e.g. a ``env['tests'][test_name]`` dict) and test run result (e.g. a ``env['test_results'][test_name]`` dict) and returns a score (instance of some subclass of :class:`~oioioi.contests.scores.ScoreValue`) and a status. Used ``environ`` keys: * ``tests`` * ``test_results`` * ``test_scorer`` Produced ``environ`` keys: * `score` and `status` keys in ``env['test_result']`` """ fun = get_object_by_dotted_name(env.get('test_scorer') or DEFAULT_TEST_SCORER) tests = env['tests'] for test_name, test_result in env['test_results'].iteritems(): score, status = fun(tests[test_name], test_result) assert isinstance(score, (types.NoneType, ScoreValue)) test_result['score'] = score and score.serialize() test_result['status'] = status return env
def grade_submission(env, **kwargs): """Grades submision on a `Job` layer. This `Handler` aggregetes score from graded groups and gets submission status from tests results. Used ``environ`` keys: * ``group_results`` * ``test_results`` * ``score_aggregator`` Produced ``environ`` keys: * ``status`` * ``score`` """ # TODO: let score_aggregator handle compilation errors if env.get('compilation_result', 'OK') != 'OK': env['score'] = None env['status'] = 'CE' return env fun = get_object_by_dotted_name( env.get('score_aggregator') or DEFAULT_SCORE_AGGREGATOR) group_results = dict( filter(lambda (name, res): res['kind'] == 'NORMAL', env['group_results'].iteritems())) score, status = fun(group_results) assert isinstance(score, (types.NoneType, ScoreValue)) env['score'] = score and score.serialize() env['status'] = status return env
def handle(self, *args, **options): if len(args) < 2: raise CommandError(_("Not enough arguments")) if len(args) > 2: raise CommandError(_("Too many arguments")) try: problem_id = int(args[0]) except ValueError: raise CommandError(_("Invalid problem id: ") + args[0]) filename = args[1] if not os.path.exists(filename): raise CommandError(_("File not found: ") + filename) try: problem = Problem.objects.get(id=problem_id) except Problem.DoesNotExist: raise CommandError(_("Problem #%d does not exist") % (problem_id,)) try: backend = \ get_object_by_dotted_name(backend_for_package(filename))() except NoBackend: raise CommandError(_("Package format not recognized")) problem = backend.simple_unpack(filename, existing_problem=problem)
def get_client(): """Constructs a Filetracker client. Needs a ``FILETRACKER_CLIENT_FACTORY`` entry in ``settings.py``, which should contain a :term:`dotted name` of a function which returns a :class:`filetracker.Client` instance. A good candidate is :func:`~oioioi.filetracker.client.media_root_factory`. The constructed client is cached. """ factory = settings.FILETRACKER_CLIENT_FACTORY if isinstance(factory, basestring): factory = get_object_by_dotted_name(factory) if not callable(factory): raise ImproperlyConfigured('The FILETRACKER_CLIENT_FACTORY setting ' 'refers to non-callable: %r' % (factory, )) client = factory() if not isinstance(client, filetracker.Client): raise ImproperlyConfigured( 'The factory pointed by ' 'FILETRACKER_CLIENT_FACTORY returned non-filetracker.Client: ' '%r' % (client, )) # Needed for oioioi.sioworkers.backends.LocalBackend so that both Django # and sioworkers use the same Filetracker client import sio.workers.ft sio.workers.ft.set_instance(client) return client
def get_zeus_server(zeus_id): """Returns ZeusServer instance for ``zeus_id``.""" server = settings.ZEUS_INSTANCES[zeus_id] # Used to inject mock instances/special handlers if server[0] == '__use_object__': return get_object_by_dotted_name(server[1])(zeus_id, server[2]) return ZeusServer(zeus_id, server)
def get_zeus_server(zeus_id): """Returns ZeusServer instance for ``zeus_id``.""" server = settings.ZEUS_INSTANCES[zeus_id] # Used to inject mock instances/special handlers if server[0] == '__use_object__': return get_object_by_dotted_name(server[1])(zeus_id, server) return ZeusServer(zeus_id, server)
def grade_tests(env, **kwargs): """Grades tests using a scoring function. The ``env['test_scorer']``, which is used by this ``Handler``, should be a path to a function which gets test definition (e.g. a ``env['tests'][test_name]`` dict) and test run result (e.g. a ``env['test_results'][test_name]`` dict) and returns a score (instance of some subclass of :class:`~oioioi.contests.scores.ScoreValue`) and a status. Used ``environ`` keys: * ``tests`` * ``test_results`` * ``test_scorer`` Produced ``environ`` keys: * `score` and `status` keys in ``env['test_result']`` """ fun = get_object_by_dotted_name( env.get('test_scorer') or DEFAULT_TEST_SCORER) tests = env['tests'] for test_name, test_result in env['test_results'].iteritems(): score, status = fun(tests[test_name], test_result) assert isinstance(score, (types.NoneType, ScoreValue)) test_result['score'] = score and score.serialize() test_result['status'] = status return env
def unpackmgr_job(env): """Creates (or modifies) a :class:`~oioioi.problems.models.Problem` instance using a package file represented by a :class:`~oioioi.problems.models.ProblemPackage`. Used ``env`` keys: ``package_id``: id of the :class:`~oioioi.problems.models.ProblemPackage` instance to process ``backend_name``: problem package backend (dotted name) to be used for unpacking ``post_upload_handlers``: a list of handler functions to be called after the new problem is created Before the handlers are called, the following ``env`` keys are produced: ``job_id``: the ``Celery`` task id ``problem_id``: id of the :class:`~oioioi.problems.models.ProblemPackage` instance, which was created or modified """ try: package = ProblemPackage.objects.get(id=env['package_id']) with package.save_operation_status(): env['job_id'] = unpackmgr_job.request.id backend = get_object_by_dotted_name(env['backend_name'])() env = backend.unpack(env) ProblemPackage.objects.get(id=env['package_id']) problem = Problem.objects.get(id=env['problem_id']) package.celery_task_id = unpackmgr_job.request.id package.problem = problem package.save() for h in env['post_upload_handlers']: handler = get_object_by_dotted_name(h) env = handler(env) except ProblemPackage.DoesNotExist: logger.warning("Problem package %s got deleted before it was " "processed.", env['package_id'])
def test_get_object_by_dotted_name(self): self.assert_(utils.get_object_by_dotted_name( 'oioioi.base.tests.TestUtils') is TestUtils) with self.assertRaises(AssertionError): utils.get_object_by_dotted_name('TestUtils') with self.assertRaisesRegexp(ImportError, 'object .* not found'): utils.get_object_by_dotted_name('oioioi.base.tests.Nonexistent') with self.assertRaises(ImportError): utils.get_object_by_dotted_name('oioioi.base.nonexistent.Foo')
def test_get_object_by_dotted_name(self): self.assert_(utils.get_object_by_dotted_name( 'oioioi.base.tests.TestUtils') is TestUtils) with self.assertRaises(AssertionError): utils.get_object_by_dotted_name('TestUtils') with self.assertRaisesRegexp(ImportError, 'object .* not found'): utils.get_object_by_dotted_name('oioioi.base.tests.Nonexistent') with self.assertRaisesRegexp(ImportError, 'object .* not found'): utils.get_object_by_dotted_name('oioioi.base.nonexistent.Foo')
def problem_sources(request): sources = [] for name in settings.PROBLEM_SOURCES: obj = get_object_by_dotted_name(name)() if isinstance(obj, ProblemSource): sources.append(obj) else: for item in obj: sources.append(item) sources = [s for s in sources if s.is_available(request)] return sources
def backend_for_package(filename, original_filename=None): """Finds a backend suitable for unpacking the given package.""" for backend_name in settings.PROBLEM_PACKAGE_BACKENDS: try: backend = get_object_by_dotted_name(backend_name)() if backend.identify(filename, original_filename): return backend except Exception: logger.warning("Backend %s probe failed", backend_name, exc_info=True) else: raise NoBackend("Problem pack format not recognized")
def render(self, context): request = context['request'] registry = self.registry.resolve(context) if not registry: registry = menu_registry if isinstance(registry, basestring): registry = get_object_by_dotted_name(registry) if not isinstance(registry, MenuRegistry): raise TemplateSyntaxError("{%% generate_menu %%} got an " "argument which is not a MenuRegistry: %r" % (registry,)) context['menu'] = registry.template_context(request) return ''
def backend_for_package(filename, original_filename=None): """Finds a backend suitable for unpacking the given package.""" for backend_name in settings.PROBLEM_PACKAGE_BACKENDS: try: backend = get_object_by_dotted_name(backend_name)() if backend.identify(filename, original_filename): return backend except Exception: logger.warning('Backend %s probe failed', backend_name, exc_info=True) raise NoBackend('Problem pack format not recognized')
def validate(self, value, model_instance): try: obj = get_object_by_dotted_name(value) except Exception: raise ValidationError(_("Object %s not found") % (value,)) superclass = self._get_superclass() if not issubclass(obj, superclass): raise ValidationError(_("%(value)s is not a %(class_name)s") % dict(value=value, class_name=superclass.__name__)) if getattr(obj, 'abstract', False): raise ValidationError(_("%s is an abstract class and cannot be " "used") % (value,))
def grade_groups(env, **kwargs): """Grades ungraded groups using a aggregating function. The ``group_scorer`` key in ``env`` should contain the path to a function which gets a list of test results (wihtout their names) and returns an aggregated score (instance of some subclass of :class:`~oioioi.contests.scores.ScoreValue`). Used ``environ`` keys: * ``tests`` * ``test_results`` * ``group_scorer`` Produced ``environ`` keys: * `score`, `max_score` and `status` keys in ``env['group_results']`` """ test_results = defaultdict(dict) for test_name, test in env['test_results'].iteritems(): group_name = env['tests'][test_name]['group'] test_results[group_name][test_name] = test env.setdefault('group_results', {}) for group_name, results in test_results.iteritems(): if group_name in env['group_results']: continue fun = get_object_by_dotted_name(env.get('group_scorer', DEFAULT_GROUP_SCORER)) score, max_score, status = fun(results) if not isinstance(score, (types.NoneType, ScoreValue)): raise TypeError("Group scorer returned %r as score, " "not None or ScoreValue" % (type(score),)) if not isinstance(max_score, (types.NoneType, ScoreValue)): raise TypeError("Group scorer returned %r as max_score, " "not None or ScoreValue" % (type(max_score),)) group_result = {} group_result['score'] = score and score.serialize() group_result['max_score'] = max_score and max_score.serialize() group_result['status'] = status one_of_tests = env['tests'][results.iterkeys().next()] if not all(env['tests'][key]['kind'] == one_of_tests['kind'] for key in results.iterkeys()): raise ValueError("Tests in group '%s' have different kinds. " "This is not supported." % (group_name,)) group_result['kind'] = one_of_tests['kind'] env['group_results'][group_name] = group_result return env
def validate(self, value, model_instance): try: obj = get_object_by_dotted_name(value) except Exception: raise ValidationError(_("Object %s not found") % (value, )) superclass = self._get_superclass() if not issubclass(obj, superclass): raise ValidationError( _("%(value)s is not a %(class_name)s") % dict(value=value, class_name=superclass.__name__)) if getattr(obj, 'abstract', False): raise ValidationError( _("%s is an abstract class and cannot be " "used") % (value, ))
def handle(self, *args, **options): if not args: raise CommandError(_("Missing argument (filename)")) if len(args) > 1: raise CommandError(_("Expected only one argument")) filename = args[0] if not os.path.exists(filename): raise CommandError(_("File not found: ") + filename) try: backend = get_object_by_dotted_name(backend_for_package(filename))() except NoBackend: raise CommandError(_("Package format not recognized")) problem = backend.simple_unpack(filename) self.stdout.write("%d\n" % (problem.id,))
def grade_groups(env, **kwargs): """Grades ungraded groups using a aggregating function. The ``group_scorer`` key in ``env`` should contain the path to a function which gets a list of test results (wihtout their names) and returns an aggregated score (instance of some subclass of :class:`~oioioi.contests.scores.ScoreValue`). Used ``environ`` keys: * ``tests`` * ``test_results`` * ``group_scorer`` Produced ``environ`` keys: * keys containg a status and a score for groups in ``env['group_results']`` """ test_results = defaultdict(dict) for test_name, test in env['test_results'].iteritems(): group_name = env['tests'][test_name]['group'] test_results[group_name][test_name] = test env.setdefault('group_results', {}) for group_name, results in test_results.iteritems(): if group_name in env['group_results']: continue fun = get_object_by_dotted_name( env.get('group_scorer', DEFAULT_GROUP_SCORER)) score, status = fun(results) if not isinstance(score, (types.NoneType, ScoreValue)): raise TypeError( "Group scorer returned %r, not None or ScoreValue" % (type(score), )) group_result = {} group_result['score'] = score and score.serialize() group_result['status'] = status one_of_tests = env['tests'][results.iterkeys().next()] if not all(env['tests'][key]['kind'] == one_of_tests['kind'] for key in results.iterkeys()): raise ValueError("Tests in group '%s' have different kinds. " "This is not supported." % (group_name, )) group_result['kind'] = one_of_tests['kind'] env['group_results'][group_name] = group_result return env
def grade_submission(env, kind='NORMAL', **kwargs): """Grades submission with specified kind of tests on a `Job` layer. If ``kind`` is None, all tests will be graded. This `Handler` aggregates score from graded groups and gets submission status from tests results. Used ``environ`` keys: * ``group_results`` * ``test_results`` * ``score_aggregator`` Produced ``environ`` keys: * ``status`` * ``score`` * ``max_score`` """ # TODO: let score_aggregator handle compilation errors if env.get('compilation_result', 'OK') != 'OK': env['score'] = None env['max_score'] = None env['status'] = 'CE' return env fun = get_object_by_dotted_name(env.get('score_aggregator') or DEFAULT_SCORE_AGGREGATOR) if kind is None: group_results = env['group_results'] else: group_results = dict((name, res) for (name, res) in env['group_results'].iteritems() if res['kind'] == kind) score, max_score, status = fun(group_results) assert isinstance(score, (types.NoneType, ScoreValue)) assert isinstance(max_score, (types.NoneType, ScoreValue)) env['score'] = score and score.serialize() env['max_score'] = max_score and max_score.serialize() env['status'] = status return env
def grade_tests(env, **kwargs): """Grades tests using a scoring function. The ``env['test_scorer']``, which is used by this ``Handler``, should be a path to a function which gets test definition (e.g. a ``env['tests'][test_name]`` dict) and test run result (e.g. a ``env['test_results'][test_name]`` dict) and returns a score (instance of some subclass of :class:`~oioioi.contests.scores.ScoreValue`) and a status. Used ``environ`` keys: * ``tests`` * ``test_results`` * ``test_scorer`` Produced ``environ`` keys: * `score`, `max_score` and `status` keys in ``env['test_result']`` """ fun = get_object_by_dotted_name(env.get('test_scorer') or DEFAULT_TEST_SCORER) tests = env['tests'] for test_name, test_result in env['test_results'].iteritems(): if tests[test_name]['to_judge']: score, max_score, status = fun(tests[test_name], test_result) assert isinstance(score, (types.NoneType, ScoreValue)) assert isinstance(max_score, (types.NoneType, ScoreValue)) test_result['score'] = score and score.serialize() test_result['max_score'] = max_score and max_score.serialize() test_result['status'] = status else: report = TestReport.objects.get( submission_report__submission__id=env['submission_id'], submission_report__status='ACTIVE', test_name=test_name) score = report.score max_score = IntegerScore(report.test_max_score) status = report.status time_used = report.time_used test_result['score'] = score and score.serialize() test_result['max_score'] = max_score and max_score.serialize() test_result['status'] = status test_result['time_used'] = time_used env['test_results'][test_name] = test_result return env
def _run_phase(env, phase, extra_kwargs={}): phaseName = phase[0] handlerName = phase[1] if len(phase) not in [2, 3]: raise TypeError("Receipt element has length neither 2 nor 3: %r" % phase) if len(phase) == 2: kwargs = {} if len(phase) == 3: kwargs = phase[2].copy() if extra_kwargs: kwargs.update(extra_kwargs) handler_func = get_object_by_dotted_name(handlerName) env = handler_func(env, **kwargs) if env is None: raise RuntimeError( 'Evaluation handler "%s" (%s) ' "forgot to return the environment." % (phaseName, handlerName) ) return env
def backend_for_package(filename, original_filename=None): """Finds a backend suitable for unpacking the given package and returns its dotted name. :param filename: a path to the processed problem package :param original_filename: the name of the package specified by the uploading user. """ for backend_name in settings.PROBLEM_PACKAGE_BACKENDS: try: backend = get_object_by_dotted_name(backend_name)() if backend.identify(filename, original_filename): return backend_name # pylint: disable=broad-except except Exception: logger.warning("Backend %s probe failed", backend_name, exc_info=True) raise NoBackend("Problem pack format not recognized")
def _run_phase(env, phase, extra_kwargs={}): phaseName = phase[0] handlerName = phase[1] if len(phase) not in [2, 3]: raise TypeError('Receipt element has length neither 2 nor 3: %r' % phase) if len(phase) == 2: kwargs = {} if len(phase) == 3: kwargs = phase[2].copy() if extra_kwargs: kwargs.update(extra_kwargs) handler_func = get_object_by_dotted_name(handlerName) env = handler_func(env, **kwargs) if env is None: raise RuntimeError('Evaluation handler "%s" (%s) ' 'forgot to return the environment.' % (phaseName, handlerName)) return env
def grade_submission(env, kind='NORMAL', **kwargs): """Grades submission with specified kind of tests on a `Job` layer. If ``kind`` is None, all tests will be graded. This `Handler` aggregates score from graded groups and gets submission status from tests results. Used ``environ`` keys: * ``group_results`` * ``test_results`` * ``score_aggregator`` Produced ``environ`` keys: * ``status`` * ``score`` """ # TODO: let score_aggregator handle compilation errors if env.get('compilation_result', 'OK') != 'OK': env['score'] = None env['status'] = 'CE' return env fun = get_object_by_dotted_name( env.get('score_aggregator') or DEFAULT_SCORE_AGGREGATOR) if kind is None: group_results = env['group_results'] else: group_results = dict( (name, res) for (name, res) in env['group_results'].iteritems() if res['kind'] == kind) score, status = fun(group_results) assert isinstance(score, (types.NoneType, ScoreValue)) env['score'] = score and score.serialize() env['status'] = status return env
def validate(self, value, model_instance): try: obj = get_object_by_dotted_name(value) except Exception, e: raise ValidationError(_("Object %s not found") % (value, ))
def controller(self): return get_object_by_dotted_name(self.controller_name)(self)
def package_backend(self): return get_object_by_dotted_name(self.package_backend_name)()
def _get_backend(): return get_object_by_dotted_name(settings.SIOWORKERS_BACKEND)()
def _get_superclass(self): if isinstance(self._superclass, basestring): self._superclass = get_object_by_dotted_name(self._superclass) return self._superclass
def validate(self, value, model_instance): try: obj = get_object_by_dotted_name(value) except Exception, e: raise ValidationError(_("Object %s not found") % (value,))
def controller(self): if not self.controller_name: return None return get_object_by_dotted_name(self.controller_name)(self)
def import_results(env, kind=None, map_to_kind=None, **kwargs): """Imports the results returned by Zeus. If ``kind`` is specified, it will only look for results with given kind. If ``map_to_kind`` is specified, all matching tests will be imported with kind replaced with ``map_to_kind``. The ``env['zeus_metadata_decoder']``, which is used by this ``Handler``, should be a path to a function which gets Zeus metadata for test (e.g. a ``env['zeus_results'][0]['metadata']`` string) and returns a dictionary which will be a base for ``test`` information (at least containing keys ``name``, ``group`` and ``max_score`` defined as below). Used ``environ`` keys: * ``zeus_results`` - populated by results fetcher Produced ``environ`` keys: * env['compilation_result'] - may be OK if the file compiled successfully or CE otherwise. * env['compilation_message'] - contains compiler stdout and stderr * ``tests`` - a dictionary mapping test names into dictionaries with following keys: ``name`` test name ``kind`` kind of the test (EXAMPLE, NORMAL) ``group`` group the test belongs to ``max_score`` maximum score the user can get for this test ``exec_time_limit`` time limit for the test (in ms) ``exec_memory_limit`` memory limit for the test (in KiB) ``zeus_metadata`` raw metadata for the test as returned by Zeus * ``test_results`` - a dictionary, mapping test names into dictionaries with the following keys: ``result_code`` test status: OK, WA, RE, ... ``result_string`` detailed supervisor information (for example, where the required and returned outputs differ) ``time_used`` total time used, in milliseconds ``zeus_test_result`` raw result returned by Zeus """ zeus_results = [r for r in env['zeus_results'] if r['report_kind'] == kind] # Assuming compilation statuses are consistent env['compilation_result'] = \ 'OK' if zeus_results[0].get('compilation_successful') else 'CE' env['compilation_message'] = zeus_results[0].get('compilation_message') if env['compilation_result'] != 'OK': return env decoder = get_object_by_dotted_name(env.get('zeus_metadata_decoder') or DEFAULT_METADATA_DECODER) tests = env.setdefault('tests', {}) test_results = env.setdefault('test_results', {}) for result in zeus_results: test = decoder(result['metadata']) error = "Not enough data decoded from: %s" % result['metadata'] assert 'name' in test, error assert 'group' in test, error assert 'max_score' in test, error if test['name'] in test_results: continue test.update({ 'zeus_metadata': result['metadata'], 'kind': map_to_kind if map_to_kind else result['report_kind'], 'exec_time_limit': result['time_limit_ms'], 'exec_memory_limit': result['memory_limit_byte'] / 1024, }) tests[test['name']] = test test_result = { 'result_code': result['status'], 'result_string': result['result_string'], 'time_used': result['execution_time_ms'], 'zeus_test_result': result, } # Fill in time_used if none given e. g. when timing machinery didn't # start yet or failed. if test_result['time_used'] is None: if result['status'] == 'TLE': test_result['time_used'] = test['exec_time_limit'] else: test_result['time_used'] = 0 test_results[test['name']] = test_result return env
def import_results(env, kind=None, map_to_kind=None, **kwargs): """Imports the results returned by Zeus. If ``kind`` is specified, it will only look for results with given kind. If ``map_to_kind`` is specified, all matching tests will be imported with kind replaced with ``map_to_kind``. The ``env['zeus_metadata_decoder']``, which is used by this ``Handler``, should be a path to a function which gets Zeus metadata for test (e.g. a ``env['zeus_results'][0]['metadata']`` string) and returns a dictionary which will be a base for ``test`` information (at least containing keys ``name``, ``group`` and ``max_score`` defined as below). Used ``environ`` keys: * ``zeus_results`` - populated by results fetcher Produced ``environ`` keys: * env['compilation_result'] - may be OK if the file compiled successfully or CE otherwise. * env['compilation_message'] - contains compiler stdout and stderr * ``tests`` - a dictionary mapping test names into dictionaries with following keys: ``name`` test name ``kind`` kind of the test (EXAMPLE, NORMAL) ``group`` group the test belongs to ``max_score`` maximum score the user can get for this test ``exec_time_limit`` time limit for the test (in ms) ``exec_memory_limit`` memory limit for the test (in KiB) ``zeus_metadata`` raw metadata for the test as returned by Zeus * ``test_results`` - a dictionary, mapping test names into dictionaries with the following keys: ``result_code`` test status: OK, WA, RE, ... ``result_string`` detailed supervisor information (for example, where the required and returned outputs differ) ``time_used`` total time used, in milliseconds ``zeus_test_result`` raw result returned by Zeus """ zeus_results = [r for r in env['zeus_results'] if r['report_kind'] == kind] # Assuming compilation statuses are consistent env['compilation_result'] = \ 'OK' if zeus_results[0].get('compilation_successful') else 'CE' env['compilation_message'] = zeus_results[0].get('compilation_message') decoder = get_object_by_dotted_name( env.get('zeus_metadata_decoder') or DEFAULT_METADATA_DECODER) tests = env.setdefault('tests', {}) test_results = env.setdefault('test_results', {}) for result in zeus_results: test = decoder(result['metadata']) assert 'name' in test assert 'group' in test assert 'max_score' in test if test['name'] in test_results: continue test.update({ 'zeus_metadata': result['metadata'], 'kind': map_to_kind if map_to_kind else result['report_kind'], 'exec_time_limit': result['time_limit_ms'], 'exec_memory_limit': result['memory_limit_byte'] / 1024, }) tests[test['name']] = test test_results[test['name']] = { 'result_code': result['status'], 'result_string': result['result_string'], 'time_used': result['execution_time_ms'], 'zeus_test_result': result, } return env