def do_run(self):
try:
self.detail = []
covered = set()
for obj in CsvSchema.LoadCSV(self.schema, self.csvdata):
# each record should be sent to all check classes, to see
# what classes it covered
for i, c in enumerate(self.check_classes):
if c(obj):
covered.add(i)
# total up score by len(covered) / total_classes
self.score = 100.0 * len(covered) / len(self.check_classes)
self.brief = lazy_gettext(
'%(rate).2f%% rules (%(cover)s out of %(total)s) covered',
cover=len(covered), total=len(self.check_classes),
rate=self.score
)
# build more detailed report
for i, c in enumerate(self.check_classes):
if i in covered:
self.detail.append(lazy_gettext(
'COVERED: %(checker)s',
checker=self.getDescription(c)
))
else:
self.detail.append(lazy_gettext(
'NOT COVERED: %(checker)s',
checker=self.getDescription(c)
))
except KeyError, ex:
raise ScorerFailure(
brief=lazy_gettext('CSV data does not match schema.'),
detail=[ex.args[0]]
)
def compile(self):
"""Validate the user submitted url address at compile stage.
The url address will be tested with the configured regex patterns
loaded from :attr:`BaseHost.compiler_params`.
Refer to :ref:`hwnetapi` for more details about the rules.
"""
if self.config['urlrule']:
p = re.compile(self.config['urlrule'])
if not p.match(self.config['remote_addr']):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'Address "%(url)s" does not match pattern "%(rule)s"',
url=self.config['remote_addr'],
rule=self.config['urlrule']))
if self.config['iprule']:
domain = urllib.splitport(
urllib.splithost(
urllib.splittype(self.config['remote_addr'])[1])[0])[0]
# get ip from domain
try:
ipaddr = socket.gethostbyname(domain)
except Exception:
logger.exception('Could not get ip address for domain "%s".' %
domain)
ipaddr = '<invalid>'
# ip not match, skip
p = re.compile(self.config['iprule'])
if not p.match(ipaddr):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'IP address "%(ip)s" does not match pattern "%(rule)s"',
ip=ipaddr,
rule=self.config['iprule']))
def do_run(self):
try:
collector = SchemaResultCollector()
self.schema.check(collector)
self.score = 100.0 * (collector.total - collector.error) / float(
collector.total)
self.brief = lazy_gettext(
'%(rate).2f%% check points (%(success)d out of %(total)d) '
'passed',
rate=self.score,
total=collector.total,
success=collector.total - collector.error,
)
self.detail = collector.errors
except:
# Why do we catch all exceptions, not the exceptions derived
# from :class:`Exception` here? Because :class:`ObjSchemaScorer`
# is usually considered not a `harmful` scorer, and the error
# messages may not be hidden to the students.
#
# If we do not catch all the exceptions, then the student
# may submit a code that `raises` something, where the evaluation
# script may be revealed.
raise ScorerFailure(brief=lazy_gettext(
'Object Structure Scorer exited with error.'),
detail=[])
def do_run(self):
try:
collector = SchemaResultCollector()
self.schema.check(collector)
self.score = 100.0 * (collector.total - collector.error) / float(
collector.total)
self.brief = lazy_gettext(
'%(rate).2f%% check points (%(success)d out of %(total)d) '
'passed',
rate=self.score,
total=collector.total,
success=collector.total - collector.error,
)
self.detail = collector.errors
except:
# Why do we catch all exceptions, not the exceptions derived
# from :class:`Exception` here? Because :class:`ObjSchemaScorer`
# is usually considered not a `harmful` scorer, and the error
# messages may not be hidden to the students.
#
# If we do not catch all the exceptions, then the student
# may submit a code that `raises` something, where the evaluation
# script may be revealed.
raise ScorerFailure(
brief=lazy_gettext(
'Object Structure Scorer exited with error.'),
detail=[]
)
def compile(self):
"""Validate the user submitted url address at compile stage.
The url address will be tested with the configured regex patterns
loaded from :attr:`BaseHost.compiler_params`.
Refer to :ref:`hwnetapi` for more details about the rules.
"""
if self.config['urlrule']:
p = re.compile(self.config['urlrule'])
if not p.match(self.config['remote_addr']):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'Address "%(url)s" does not match pattern "%(rule)s"',
url=self.config['remote_addr'], rule=self.config['urlrule']
))
if self.config['iprule']:
domain = urllib.splitport(
urllib.splithost(
urllib.splittype(self.config['remote_addr'])[1]
)[0]
)[0]
# get ip from domain
try:
ipaddr = socket.gethostbyname(domain)
except Exception:
logger.exception(
'Could not get ip address for domain "%s".' % domain)
ipaddr = '<invalid>'
# ip not match, skip
p = re.compile(self.config['iprule'])
if not p.match(ipaddr):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'IP address "%(ip)s" does not match pattern "%(rule)s"',
ip=ipaddr, rule=self.config['iprule']
))
def __init__(self, score):
super(JavaScore, self).__init__(lazy_gettext("Functionality Scorer"))
self.score = score
self.brief = lazy_gettext("%(rate).2f%% rules (%(cover)s out of %(total)s) covered", cover=1, total=1, rate=100)
self.detail = [
lazy_gettext("%(rate).2f%% rules (%(cover)s out of %(total)s) covered", cover=1, total=1, rate=100)
]
def do_run(self):
if type(self.filelist) == type("a"):
ph_out = self.filelist
self.detail = []
warning = 0
l = ph_out.split("\n")
for x in l:
if x[:6] == "[WARN]":
warning += 1
self.detail.append(x)
self.score = 100.0 - warning * self.errcost
if self.score < 0.0:
self.score = 0.0
total_file = 1
if warning > 0:
self.brief = lazy_gettext(
"%(trouble)d problem(s) found in %(file)d file(s)", trouble=warning, file=total_file
)
else:
self.brief = lazy_gettext("All files passed Google code style check")
if self.logs != None:
self.logs.saveCodeStyle(
str(self.score) + "\n" + str(self.brief) + "\n" + GetTextStringList(self.detail)
)
return
guide = pep8.StyleGuide()
guide.options.show_source = True
guide.options.report = Pep8DetailReport(guide.options)
result = guide.check_files(self.filelist)
# Each error consumes 1 point.
errcount = result.count_errors()
self.score = 100.0 - errcount * self.errcost
if self.score < 0.0:
self.score = 0.0
# format the brief report
total_file = len(self.filelist)
if errcount > 0:
self.brief = lazy_gettext(
"%(trouble)d problem(s) found in %(file)d file(s)", trouble=errcount, file=total_file
)
else:
self.brief = lazy_gettext("All files passed PEP8 code style check")
# format detailed reports
self.detail = result.build_report()
if self.logs != None:
self.logs.saveCodeStyle(str(self.score) + "\n" + str(self.brief) + "\n" + GetTextStringList(self.detail))
def addError(self, test, err):
super(UnitTestScorerDetailResult, self).addError(test, err)
self.details.append(lazy_gettext(
'ERROR: %(test)s.\n%(error)s',
test=self.getDescription(test),
error=self._exc_info_to_string(err, test)
))
def do_run(self):
# First, load the data, and format the error message if schema
# is not matched.
try:
loaded_data = list(CsvSchema.LoadCSV(self.schema, self.csvdata))
except KeyError, ex:
raise ScorerFailure(brief=lazy_gettext("CSV data does not match schema."), detail=[ex.args[0]])
def __init__(self, schema, csvdata, check_classes=None):
super(InputClassScorer, self).__init__(
name=lazy_gettext('InputClass Scorer'),
schema=schema,
csvdata=csvdata,
check_classes=check_classes,
)
def __init__(self, schema, csvdata, check_classes=None):
super(BoundaryValueScorer, self).__init__(
name=lazy_gettext('BoundaryValue Scorer'),
schema=schema,
csvdata=csvdata,
check_classes=check_classes,
)
def __init__(self, schema, csvdata, check_classes=None):
super(InputClassScorer, self).__init__(
name=lazy_gettext('InputClass Scorer'),
schema=schema,
csvdata=csvdata,
check_classes=check_classes,
)
def addSkip(self, test, reason):
super(UnitTestScorerDetailResult, self).addSkip(test, reason)
self.details.append(lazy_gettext(
'SKIP: %(test)s: %(reason)s.',
test=self.getDescription(test),
reason=reason
))
def addExpectedFailure(self, test, err):
super(UnitTestScorerDetailResult, self).addExpectedFailure(test, err)
self.details.append(lazy_gettext(
'EXPECTED FAIL: %(test)s.\n%(error)s',
test=self.getDescription(test),
error=self._exc_info_to_string(err, test)
))
def __init__(self, schema, csvdata, check_classes=None):
super(BoundaryValueScorer, self).__init__(
name=lazy_gettext('BoundaryValue Scorer'),
schema=schema,
csvdata=csvdata,
check_classes=check_classes,
)
def do_run(self):
if type(self.schema) == type("a"):
ph_out = self.schema
l = ph_out.split("\n")
self.detail = l[:-2]
total = int(l[-2])
error = int(l[-1])
self.score = 100.0 * (total - error) / float(total)
self.brief = lazy_gettext(
"%(rate).2f%% check points (%(success)d out of %(total)d) " "passed",
rate=self.score,
total=total,
success=total - error,
)
if self.logs != None:
self.logs.saveScheme(str(self.score) + "\n" + str(self.brief) + "\n" + GetTextStringList(self.detail))
return
try:
collector = SchemaResultCollector()
self.schema.check(collector)
self.score = 100.0 * (collector.total - collector.error) / float(collector.total)
self.brief = lazy_gettext(
"%(rate).2f%% check points (%(success)d out of %(total)d) " "passed",
rate=self.score,
total=collector.total,
success=collector.total - collector.error,
)
self.detail = collector.errors
if self.logs != None:
self.logs.saveScheme(str(self.score) + "\n" + str(self.brief) + "\n" + GetTextStringList(self.detail))
except:
# Why do we catch all exceptions, not the exceptions derived
# from :class:`Exception` here? Because :class:`ObjSchemaScorer`
# is usually considered not a `harmful` scorer, and the error
# messages may not be hidden to the students.
#
# If we do not catch all the exceptions, then the student
# may submit a code that `raises` something, where the evaluation
# script may be revealed.
if self.logs != None:
self.logs.saveScheme(lazy_gettext("Object Structure Scorer exited with error."))
raise ScorerFailure(brief=lazy_gettext("Object Structure Scorer exited with error."), detail=[])
def __init__(self, suite, filelist, stmt_weight=0.5, branch_weight=0.5):
super(CoverageScorer, self).__init__(lazy_gettext('Coverage Scorer'))
self.suite = suite
self.brief = []
self.filelist = list(filelist)
self.stmt_weight = stmt_weight
self.branch_weight = branch_weight
def parseString(self, value):
try:
return self.fromString(value)
except Exception:
raise ValueError(lazy_gettext(
'Cannot convert "%(value)s" to %(type)s.',
value=value, type=self.__class__.__name__
))
def check_require(self, collector):
"""Check whether this schema is marked as `REQUIRE` but the object
does not exist. All the children schema will also be notified.
"""
if self.exist_rule == SchemaExistRule.REQUIRE and not self.exist:
collector.addError(lazy_gettext(
'%(schema)s is required but the object does not exist or '
'could not be loaded.',
schema=self.get_description()
))
elif self.exist_rule == SchemaExistRule.DENY and self.exist:
collector.addError(lazy_gettext(
'%(schema)s is denied but the object exists.',
schema=self.get_description()
))
else:
collector.addSuccess()
def __init__(self, filelist, skipfile=None, errcost=10.0):
super(CodeStyleScorer, self).__init__(lazy_gettext('CodeStyle Scorer'))
skipfile = skipfile or (lambda path: False)
is_pyfile = lambda p: (p[-3:].lower() == '.py')
self.filelist = [p for p in filelist
if not skipfile(p) and is_pyfile(p)]
self.errcost = errcost
def __init__(self, suite, filelist, stmt_weight=0.5, branch_weight=0.5):
super(CoverageScorer, self).__init__(lazy_gettext('Coverage Scorer'))
self.suite = suite
self.brief = []
self.filelist = list(filelist)
self.stmt_weight = stmt_weight
self.branch_weight = branch_weight
def api_handin_proclog(uuid):
"""Store the process outputs for a given submission.
This api view is usually requested after the reports of the corresponding
submission has been stored, so it would not change either the score or
the state of the submission.
If the submission state is still `Pending` or `Running`, indicating that
the reports have not been stored (probably the process exited abnormally
before report the score), the state will be updated to `Rejected`.
This view will compare `uuid` in POST object to the `uuid` argument.
If they are not equal, the operation will be rejected, since it is
likely to be an attack.
:route: /api/handin/proclog/<uuid>/
:payload:
.. code-block:: python
{"uuid": uuid of submission,
"exitcode": The exitcode of the process,
"stdout": The standard output of the process,
"stderr": The standard error output of the process}
:param uuid: The uuid of submission.
:type uuid: :class:`str`
:return: ``OK`` if succeeded, error messages otherwise.
"""
obj = request.payload
# check uuid, so that we can prevent replay attack
if obj['uuid'] != uuid:
return 'uuid mismatch, do not attack'
# load the handin object, and report error if not exist
handin = Handin.query.filter(Handin.uuid == uuid).first()
if not handin:
return 'requested submission not found'
# if handin.state != 'Accepted' and handin.state != 'Rejected',
# the process must have exited without report the score.
# mark such handin as "Rejected"
if handin.state != 'Accepted' and handin.state != 'Rejected':
handin.state = 'Rejected'
handin.result = lazy_gettext('Process exited before reporting score.')
handin.partials = []
try:
handin.exitcode = obj['exitcode']
handin.stdout = obj['stdout']
handin.stderr = obj['stderr']
db.session.commit()
except Exception:
app.logger.exception('Cannot log proccess of submission(%s).' % uuid)
return 'update database failed'
return 'OK'
def api_handin_proclog(uuid):
"""Store the process outputs for a given submission.
This api view is usually requested after the reports of the corresponding
submission has been stored, so it would not change either the score or
the state of the submission.
If the submission state is still `Pending` or `Running`, indicating that
the reports have not been stored (probably the process exited abnormally
before report the score), the state will be updated to `Rejected`.
This view will compare `uuid` in POST object to the `uuid` argument.
If they are not equal, the operation will be rejected, since it is
likely to be an attack.
:route: /api/handin/proclog/<uuid>/
:payload:
.. code-block:: python
{"uuid": uuid of submission,
"exitcode": The exitcode of the process,
"stdout": The standard output of the process,
"stderr": The standard error output of the process}
:param uuid: The uuid of submission.
:type uuid: :class:`str`
:return: ``OK`` if succeeded, error messages otherwise.
"""
obj = request.payload
# check uuid, so that we can prevent replay attack
if obj['uuid'] != uuid:
return 'uuid mismatch, do not attack'
# load the handin object, and report error if not exist
handin = Handin.query.filter(Handin.uuid == uuid).first()
if not handin:
return 'requested submission not found'
# if handin.state != 'Accepted' and handin.state != 'Rejected',
# the process must have exited without report the score.
# mark such handin as "Rejected"
if handin.state != 'Accepted' and handin.state != 'Rejected':
handin.state = 'Rejected'
handin.result = lazy_gettext('Process exited before reporting score.')
handin.partials = []
try:
handin.exitcode = obj['exitcode']
handin.stdout = obj['stdout']
handin.stderr = obj['stderr']
db.session.commit()
except Exception:
app.logger.exception('Cannot log proccess of submission(%s).' % uuid)
return 'update database failed'
return 'OK'
def do_run(self):
# First, load the data, and format the error message if schema
# is not matched.
try:
loaded_data = list(CsvSchema.LoadCSV(self.schema, self.csvdata))
except KeyError, ex:
raise ScorerFailure(
brief=lazy_gettext('CSV data does not match schema.'),
detail=[ex.args[0]])
def __init__(self, suite):
super(UnitTestScorer, self).__init__(
lazy_gettext('Functionality Scorer'))
#: Store the testing suite. If it is a :func:`callable` object but not
#: a #: :class:`~unittest.suite.TestSuite`, execute it to get the real
#: testing suite. Keep the :attr:`suite` lazy can prevent the scorer
#: from exploits.
self.suite = suite
def __init__(self, suite):
super(UnitTestScorer,
self).__init__(lazy_gettext('Functionality Scorer'))
#: Store the testing suite. If it is a :func:`callable` object but not
#: a #: :class:`~unittest.suite.TestSuite`, execute it to get the real
#: testing suite. Keep the :attr:`suite` lazy can prevent the scorer
#: from exploits.
self.suite = suite
def __init__(self, filelist, skipfile=None, errcost=10.0):
super(CodeStyleScorer, self).__init__(lazy_gettext('CodeStyle Scorer'))
skipfile = skipfile or (lambda path: False)
is_pyfile = lambda p: (p[-3:].lower() == '.py')
self.filelist = [
p for p in filelist if not skipfile(p) and is_pyfile(p)
]
self.errcost = errcost
def __init__(self, filelist, skipfile=None, errcost=10.0, logs=None):
super(CodeStyleScorer, self).__init__(lazy_gettext("CodeStyle Scorer"), logs)
skipfile = skipfile or (lambda path: False)
is_pyfile = lambda p: (p[-3:].lower() == ".py")
if type(filelist) == type("a"):
self.filelist = filelist
else:
self.filelist = [p for p in filelist if not skipfile(p) and is_pyfile(p)]
self.errcost = errcost
def do_run(self):
guide = pep8.StyleGuide()
guide.options.show_source = True
guide.options.report = Pep8DetailReport(guide.options)
result = guide.check_files(self.filelist)
# Each error consumes 1 point.
errcount = result.count_errors()
self.score = 100.0 - errcount * self.errcost
if self.score < 0.0:
self.score = 0.0
# format the brief report
total_file = len(self.filelist)
if errcount > 0:
self.brief = lazy_gettext(
'%(trouble)d problem(s) found in %(file)d file(s)',
trouble=errcount, file=total_file
)
else:
self.brief = lazy_gettext('All files passed PEP8 code style check')
# format detailed reports
self.detail = result.build_report()
def do_run(self):
guide = pep8.StyleGuide()
guide.options.show_source = True
guide.options.report = Pep8DetailReport(guide.options)
result = guide.check_files(self.filelist)
# Each error consumes 1 point.
errcount = result.count_errors()
self.score = 100.0 - errcount * self.errcost
if self.score < 0.0:
self.score = 0.0
# format the brief report
total_file = len(self.filelist)
if errcount > 0:
self.brief = lazy_gettext(
'%(trouble)d problem(s) found in %(file)d file(s)',
trouble=errcount,
file=total_file)
else:
self.brief = lazy_gettext('All files passed PEP8 code style check')
# format detailed reports
self.detail = result.build_report()
def do_run(self):
self.suite = load_suite(self.suite)
# get the result of unittest
result = UnitTestScorerDetailResult()
self.suite.run(result)
# format score and reports according to unittest result
total = self.suite.countTestCases()
errors, failures = map(len, (result.errors, result.failures))
# give out a score according to the above statistics
success = total - (errors + failures)
if total > 0:
self.score = 100.0 * success / total
else:
self.score = 100.0
# format the brief report
self.brief = lazy_gettext(
'%(rate).2f%% tests (%(success)d out of %(total)d) passed',
rate=self.score, total=total, time=self.time, success=success
)
# format the detailed report
self.detail = result.details
def do_run(self):
self.suite = load_suite(self.suite)
# get the result of unittest
result = UnitTestScorerDetailResult()
self.suite.run(result)
# format score and reports according to unittest result
total = self.suite.countTestCases()
errors, failures = map(len, (result.errors, result.failures))
# give out a score according to the above statistics
success = total - (errors + failures)
if total > 0:
self.score = 100.0 * success / total
else:
self.score = 100.0
# format the brief report
self.brief = lazy_gettext(
'%(rate).2f%% tests (%(success)d out of %(total)d) passed',
rate=self.score,
total=total,
time=self.time,
success=success)
# format the detailed report
self.detail = result.details
def LoadCSV(cls, iterable):
"""Get iterable objects from given line `iterable` object."""
rdr = csv.reader(iterable)
# parse the header line
headers = {k: i for i, k in enumerate(next(rdr))}
field_getter = {}
for k, v in cls.__dict__.iteritems():
if isinstance(v, CsvField):
field_name = v.name if v.name else k
if field_name in headers:
# set the getter to fetch Nth column of a row
# where N = headers[k]
field_getter[k] = (
lambda row, key=field_name, col=v: (
col.parseString(row[headers[key]])
)
)
elif v.has_default:
# not exist in CSV, if has default, use default value
field_getter[k] = lambda row, val=v: val.default
else:
# not exist, no default, raise KeyError
raise KeyError(lazy_gettext(
'Field "%(field)s" not found in CSV data.',
field=field_name
))
# Yield object from CSV one by one
for row in rdr:
if not row:
continue
obj = cls()
for f, g in field_getter.iteritems():
setattr(obj, f, g(row))
yield obj
def __init__(self, **kwargs):
super(RuntimeFileCopyFailure, self).__init__(lazy_gettext(
"Couldn't copy runtime files, please contact TA."
), **kwargs)
brief=lazy_gettext('CSV data does not match schema.'),
detail=[ex.args[0]])
# Next, check the loaded data
self.detail = []
covered = set()
for obj in loaded_data:
# each record should be sent to all check classes, to see
# what classes it covered
for i, c in enumerate(self.check_classes):
if c(obj):
covered.add(i)
# total up score by len(covered) / total_classes
self.score = 100.0 * len(covered) / len(self.check_classes)
self.brief = lazy_gettext(
'%(rate).2f%% rules (%(cover)s out of %(total)s) covered',
cover=len(covered),
total=len(self.check_classes),
rate=self.score)
# build more detailed report
for i, c in enumerate(self.check_classes):
if i in covered:
self.detail.append(
lazy_gettext('COVERED: %(checker)s',
checker=self.getDescription(c)))
else:
self.detail.append(
lazy_gettext('NOT COVERED: %(checker)s',
checker=self.getDescription(c)))
class InputClassScorer(InputDataScorer):
def addUnexpectedSuccess(self, test):
super(UnitTestScorerDetailResult, self).addUnexpectedSuccess(test)
self.details.append(lazy_gettext(
'UNEXPECTED SUCCESS: %(test)s.',
test=self.getDescription(test)
))
def __init__(self, schema):
super(ObjSchemaScorer, self) .__init__(
lazy_gettext('Object Structure Scorer')
)
self.schema = schema
def addSuccess(self, test):
super(UnitTestScorerDetailResult, self).addSuccess(test)
self.details.append(
lazy_gettext('PASSED: %(test)s.', test=self.getDescription(test)))
def __init__(self, **kwargs):
super(ExtractFileFailure, self).__init__(lazy_gettext(
"Couldn't extract your archive file."
), **kwargs)
def __init__(self, **kwargs):
super(RunnerPermissionError, self).__init__(lazy_gettext(
'File permissions of the runner is wrong.'
), **kwargs)
def __init__(self, **kwargs):
super(SpawnProcessFailure, self).__init__(lazy_gettext(
"Couldn't start submitted program."
), **kwargs)
def do_run(self):
def safe_divide(a, b, default=1.0):
if b > 0:
return float(a) / float(b)
return default
cov = coverage(branch=True)
cov.start()
self.suite = load_suite(self.suite)
# Run the test suite
# the `result` is now ignored, but we can get use of it if necessary
result = UnitTestScorerDetailResult()
self.suite.run(result)
cov.stop()
# the 1st part: total view of the coverage stats
self.detail = ['']
total_cov = []
# statement coverage rate
total_exec = total_miss = 0
total_branch = total_taken = total_partial = total_notaken = 0
for filename in self.filelist:
# get the analysis on given filename
ana = cov._analyze(filename)
# gather statement coverage on this file
exec_stmt = set(ana.statements)
miss_stmt = set(ana.missing)
total_exec += len(exec_stmt)
total_miss += len(miss_stmt)
# gather branch coverage on this file
# branch: {lineno: (total_exit, taken_exit)}
branch = ana.branch_stats()
file_branch = len(branch)
file_taken = len([b for b in branch.itervalues() if b[0] == b[1]])
file_notaken = len(
[b for b in branch.itervalues() if b[0] != b[1] and b[1] == 0])
file_partial = file_branch - file_taken - file_notaken
# add the file stats to total coverage results
total_cov.append(
'%(file)s, %(stmt)d, %(stmt_taken)d, %(stmt_cov).2f%%, '
'%(branch)d, %(branch_taken)d, %(branch_partial)d, '
'%(branch_cov).2f%%, %(branch_partial_cov).2f%%' % {
'file':
filename,
'stmt':
len(exec_stmt),
'stmt_taken':
len(exec_stmt) - len(miss_stmt),
'stmt_cov':
100.0 * safe_divide(
len(exec_stmt) - len(miss_stmt), len(exec_stmt)),
'branch':
file_branch,
'branch_taken':
file_taken,
'branch_partial':
file_partial,
'branch_cov':
100.0 * safe_divide(file_taken, file_branch),
'branch_partial_cov':
100.0 * safe_divide(file_partial, file_branch, default=0.0)
})
# apply file branch to global
total_branch += file_branch
total_taken += file_taken
total_partial += file_partial
total_notaken += file_notaken
# gather all source lines into detail report
stmt_text = []
branch_text = []
with open(filename, 'rb') as fsrc:
for i, s in enumerate(fsrc, 1):
# first, format statement cover report
if i in miss_stmt:
stmt_text.append('- %s' % s.rstrip())
elif i in exec_stmt:
stmt_text.append('+ %s' % s.rstrip())
else:
stmt_text.append(' %s' % s.rstrip())
# next, format branch cover report
branch_exec = branch.get(i, None)
if not branch_exec:
branch_text.append(' %s' % s.rstrip())
elif branch_exec[1] == branch_exec[0]:
# branch taken
branch_text.append('+ %s' % s.rstrip())
elif branch_exec[1] == 0:
# branch not taken
branch_text.append('- %s' % s.rstrip())
else:
# branch partial taken
branch_text.append('* %s' % s.rstrip())
# compose final detail
stmt_text = '\n'.join(stmt_text)
branch_text = '\n'.join(branch_text)
# the statement coverage
self.detail.append(
lazy_gettext(
'%(filename)s: %(miss)d statement(s) not covered.\n'
'%(sep)s\n'
'%(source)s',
filename=filename,
sep='-' * 70,
miss=len(miss_stmt),
source=stmt_text))
# the branch coverage
self.detail.append(
lazy_gettext(
'%(filename)s: '
'%(partial)d branch(es) partially taken and '
'%(notaken)d branch(es) not taken.\n'
'%(sep)s\n'
'%(source)s',
filename=filename,
sep='-' * 70,
miss=len(miss_stmt),
source=branch_text,
taken=file_taken,
notaken=file_notaken,
partial=file_partial))
self.stmt_cover = 100.0 - 100.0 * safe_divide(total_miss, total_exec)
self.branch_cover = 100.0 * safe_divide(total_taken, total_branch)
self.branch_partial = 100.0 * safe_divide(
total_partial, total_branch, default=0.0)
# Add final total report
self.detail[0] = lazy_gettext(
'Coverage Results:\n'
'%(delim1)s\n'
'file, stmts, taken, covered, branches, taken, partially taken, '
'covered, partially covered\n'
'%(delim2)s\n'
'%(detail)s\n'
'%(delim2)s\n'
'total, %(stmt)d, %(stmt_taken)d, %(stmt_cov).2f%%, '
'%(branch)d, %(branch_taken)d, %(branch_partial)d, '
'%(branch_cov).2f%%, %(branch_partial_cov).2f%%',
delim1='=' * 70,
delim2='-' * 70,
detail='\n'.join(total_cov),
stmt=total_exec,
stmt_taken=total_exec - total_miss,
stmt_cov=self.stmt_cover,
branch=total_branch,
branch_taken=total_taken,
branch_partial=total_partial,
branch_cov=self.branch_cover,
branch_partial_cov=self.branch_partial,
)
# final score
stmt_score = self.stmt_cover * self.stmt_weight
full_branch_score = self.branch_cover * self.branch_weight
partial_branch_score = self.branch_partial * self.branch_weight * 0.5
self.score = stmt_score + full_branch_score + partial_branch_score
self.brief = lazy_gettext(
'%(stmt).2f%% statements covered (%(stmt_score).2f pts), '
'%(branch).2f%% branches fully covered (%(branch_score).2f pts) '
'and '
'%(partial).2f%% partially covered (%(partial_score).2f pts).',
stmt=self.stmt_cover,
branch=self.branch_cover,
partial=self.branch_partial,
stmt_score=stmt_score,
branch_score=full_branch_score,
partial_score=partial_branch_score,
)
def addUnexpectedSuccess(self, test):
super(UnitTestScorerDetailResult, self).addUnexpectedSuccess(test)
self.details.append(
lazy_gettext('UNEXPECTED SUCCESS: %(test)s.',
test=self.getDescription(test)))
def __init__(self, **kwargs):
super(SpawnProcessFailure,
self).__init__(lazy_gettext("Couldn't start submitted program."),
**kwargs)
def run_handin(handler, handid, hwid):
"""Common pattern to run a submission. Its main function is to
glue :class:`~railgun.runner.handin.BaseHandin`,
:class:`~railgun.runner.host.BaseHost` and
:class:`~railgun.runner.apiclient.ApiClient` together.
It is guaranteed that all errors are handled and logged correctly in this
method.
:param handler: A factory to create a
:class:`~railgun.runner.handin.BaseHandin` handler object.
:param handid: The uuid of this submission.
:type handid: :class:`str`
:param hwid: The uuid of the homework.
:type hwid: :class:`str`
"""
# Create the api client, we may use it once or twice
api = ApiClient(runconfig.WEBSITE_API_BASEURL)
# Immediately report error if permcheck has error
if permcheck.checker.has_error():
report_error(handid, RunnerPermissionError())
return
try:
report_start(handid)
# create and launch this handler
if callable(handler):
handler = handler()
exitcode, stdout, stderr = handler.execute()
# try to convert stdout & stderr to unicode in UTF-8 encoding
# if not success, report the client has produced non UTF-8 output
try:
stdout = unicode(stdout, 'utf-8')
stderr = unicode(stderr, 'utf-8')
except UnicodeError:
# This routine will terminate the try-catch structure so that
# we must report the exitcode earlier as well.
api.proclog(handid, exitcode, None, None)
raise NonUTF8OutputError()
# log the handin execution
if exitcode != 0:
logger.warning(
'Submission[%(handid)s] of hw[%(hwid)s]: Error.\n'
' stdout: %(stdout)s\n'
' stderr: %(stderr)s' % {
'handid': handid,
'hwid': hwid,
'stdout': repr(stdout),
'stderr': repr(stderr)
})
# Report failure if exitcode != 0. In this case the host itself may
# not have the chance to report handin scores
if exitcode != 0:
# We do not raise RunnerError here, because under this situation,
# we must have logged such exception, and do not want to log
# again.
score = HwScore(
False,
lazy_gettext('Exitcode %(exitcode)s != 0.', exitcode=exitcode))
api.report(handid, score)
# Update exitcode, stdout and stderr here, which cannot be set in
# the host itself.
#
# This process may also change Handin.state, if previous process
# exit with code 0 before it reported the score. See website/api.py
# for more details.
api.proclog(handid, exitcode, stdout, stderr)
# Log that we've succesfully done this job.
logger.info('Submission[%(handid)s] of hw[%(hwid)s]: OK.' % {
'handid': handid,
'hwid': hwid
})
except RunnerError, ex:
# RunnerError is logically OK and sent to client only.
# So we just log the message of this exception, not exception detail.
logger.warning('Submission[%(handid)s] of hw[%(hwid)s]: %(message)s.' %
{
'handid': handid,
'hwid': hwid,
'message': ex.message
})
report_error(handid, ex)
def addExpectedFailure(self, test, err):
super(UnitTestScorerDetailResult, self).addExpectedFailure(test, err)
self.details.append(
lazy_gettext('EXPECTED FAIL: %(test)s.\n%(error)s',
test=self.getDescription(test),
error=self._exc_info_to_string(err, test)))
def __init__(self, **kwargs):
super(RuntimeFileCopyFailure, self).__init__(
lazy_gettext("Couldn't copy runtime files, please contact TA."),
**kwargs)
def __init__(self, **kwargs):
super(ArchiveContainTooManyFileError,
self).__init__(lazy_gettext("Archive contains too many files."),
**kwargs)
def do_run(self):
def safe_divide(a, b, default=1.0):
if b > 0:
return float(a) / float(b)
return default
cov = coverage(branch=True)
cov.start()
self.suite = load_suite(self.suite)
# Run the test suite
# the `result` is now ignored, but we can get use of it if necessary
result = UnitTestScorerDetailResult()
self.suite.run(result)
cov.stop()
# the 1st part: total view of the coverage stats
self.detail = ['']
total_cov = []
# statement coverage rate
total_exec = total_miss = 0
total_branch = total_taken = total_partial = total_notaken = 0
for filename in self.filelist:
# get the analysis on given filename
ana = cov._analyze(filename)
# gather statement coverage on this file
exec_stmt = set(ana.statements)
miss_stmt = set(ana.missing)
total_exec += len(exec_stmt)
total_miss += len(miss_stmt)
# gather branch coverage on this file
# branch: {lineno: (total_exit, taken_exit)}
branch = ana.branch_stats()
file_branch = len(branch)
file_taken = len([b for b in branch.itervalues() if b[0] == b[1]])
file_notaken = len([b for b in branch.itervalues()
if b[0] != b[1] and b[1] == 0])
file_partial = file_branch - file_taken - file_notaken
# add the file stats to total coverage results
total_cov.append(
'%(file)s, %(stmt)d, %(stmt_taken)d, %(stmt_cov).2f%%, '
'%(branch)d, %(branch_taken)d, %(branch_partial)d, '
'%(branch_cov).2f%%, %(branch_partial_cov).2f%%' % {
'file': filename,
'stmt': len(exec_stmt),
'stmt_taken': len(exec_stmt) - len(miss_stmt),
'stmt_cov': 100.0 * safe_divide(
len(exec_stmt) - len(miss_stmt), len(exec_stmt)),
'branch': file_branch,
'branch_taken': file_taken,
'branch_partial': file_partial,
'branch_cov': 100.0 * safe_divide(file_taken, file_branch),
'branch_partial_cov': 100.0 * safe_divide(
file_partial, file_branch, default=0.0)
}
)
# apply file branch to global
total_branch += file_branch
total_taken += file_taken
total_partial += file_partial
total_notaken += file_notaken
# gather all source lines into detail report
stmt_text = []
branch_text = []
with open(filename, 'rb') as fsrc:
for i, s in enumerate(fsrc, 1):
# first, format statement cover report
if i in miss_stmt:
stmt_text.append('- %s' % s.rstrip())
elif i in exec_stmt:
stmt_text.append('+ %s' % s.rstrip())
else:
stmt_text.append(' %s' % s.rstrip())
# next, format branch cover report
branch_exec = branch.get(i, None)
if not branch_exec:
branch_text.append(' %s' % s.rstrip())
elif branch_exec[1] == branch_exec[0]:
# branch taken
branch_text.append('+ %s' % s.rstrip())
elif branch_exec[1] == 0:
# branch not taken
branch_text.append('- %s' % s.rstrip())
else:
# branch partial taken
branch_text.append('* %s' % s.rstrip())
# compose final detail
stmt_text = '\n'.join(stmt_text)
branch_text = '\n'.join(branch_text)
# the statement coverage
self.detail.append(lazy_gettext(
'%(filename)s: %(miss)d statement(s) not covered.\n'
'%(sep)s\n'
'%(source)s',
filename=filename, sep='-' * 70, miss=len(miss_stmt),
source=stmt_text
))
# the branch coverage
self.detail.append(lazy_gettext(
'%(filename)s: '
'%(partial)d branch(es) partially taken and '
'%(notaken)d branch(es) not taken.\n'
'%(sep)s\n'
'%(source)s',
filename=filename, sep='-' * 70, miss=len(miss_stmt),
source=branch_text, taken=file_taken, notaken=file_notaken,
partial=file_partial
))
self.stmt_cover = 100.0 - 100.0 * safe_divide(total_miss, total_exec)
self.branch_cover = 100.0 * safe_divide(total_taken, total_branch)
self.branch_partial = 100.0 * safe_divide(
total_partial, total_branch, default=0.0)
# Add final total report
self.detail[0] = lazy_gettext(
'Coverage Results:\n'
'%(delim1)s\n'
'file, stmts, taken, covered, branches, taken, partially taken, '
'covered, partially covered\n'
'%(delim2)s\n'
'%(detail)s\n'
'%(delim2)s\n'
'total, %(stmt)d, %(stmt_taken)d, %(stmt_cov).2f%%, '
'%(branch)d, %(branch_taken)d, %(branch_partial)d, '
'%(branch_cov).2f%%, %(branch_partial_cov).2f%%',
delim1='=' * 70,
delim2='-' * 70,
detail='\n'.join(total_cov),
stmt=total_exec,
stmt_taken=total_exec - total_miss,
stmt_cov=self.stmt_cover,
branch=total_branch,
branch_taken=total_taken,
branch_partial=total_partial,
branch_cov=self.branch_cover,
branch_partial_cov=self.branch_partial,
)
# final score
stmt_score = self.stmt_cover * self.stmt_weight
full_branch_score = self.branch_cover * self.branch_weight
partial_branch_score = self.branch_partial * self.branch_weight * 0.5
self.score = stmt_score + full_branch_score + partial_branch_score
self.brief = lazy_gettext(
'%(stmt).2f%% statements covered (%(stmt_score).2f pts), '
'%(branch).2f%% branches fully covered (%(branch_score).2f pts) '
'and '
'%(partial).2f%% partially covered (%(partial_score).2f pts).',
stmt=self.stmt_cover,
branch=self.branch_cover,
partial=self.branch_partial,
stmt_score=stmt_score,
branch_score=full_branch_score,
partial_score=partial_branch_score,
)
class InputDataScorer(Scorer):
"""The base class for input data scorers.
Input data scorers are mainly used in BlackBox testing homework.
The students may provide some structured data in CSV format,
and we want to check whether the data covers all the equivalent
classes and boundary values.
To achieve that goal, I introduced :class:`InputDataScorer`.
It can takes a set of methods as `condition` validators, where
each methods only returns :data:`True` on a certain class of
input data or boundary value.
Then the student submitted data will be checked by all the
`condition` validators one row after another. The scorer will
count the number of validators who have ever reported :data:`True`,
and give the score according to that portion.
:param name: The name of this scorer, should be set by derived classes.
:type name: :class:`~railgun.common.lazy_i18n.GetTestString`
:param schema: The schema for this scorer to parse csv data.
:type schema: :class:`~railgun.common.csvdata.CsvSchema`
:param csvdata: Iterable object over :class:`str`, each representing a
row in the csv data. Usually a :class:`file` object.
:type csvdata: :class:`object`
:param check_classes: The initial list of input data validators.
:type check_classes: :class:`list` of :func:`callable` objects
"""
def __init__(self, name, schema, csvdata, check_classes=None):
"""Construct a new `InputClassScorer` on given `csvdata`, checked by
rules defined in `check_classes`."""
super(InputDataScorer, self).__init__(name)
#: Store the :class:`~railgun.common.csvdata.CsvSchema`.
self.schema = schema
#: Rows of csv data.
self.csvdata = csvdata
#: The input data validators.
self.check_classes = check_classes or []
def empty(self):
"""Whether or not this scorer has no input data validator?"""
return not self.check_classes
def getDescription(self, check_class):
"""Get the description for given validator.
The `description` attribute of the given validator, or `__name__`
if `description` doesn't exist, or the string representation of the
given validator if `__name__` doesn't exist either.
:param check_class: The :func:`callable` data validator.
:return: The description of given validator.
"""
if hasattr(check_class, 'description'):
return getattr(check_class, 'description')
if hasattr(check_class, '__name__'):
return getattr(check_class, '__name__')
return str(check_class)
def rule(self, description):
"""Make a decorator to :func:`callable` objects which will add
`description` attribute to the given method, and will add that
method to :attr:`check_classes`.
Usage::
@scorer.rule('a >= 1 and b <= 2')
def a_must_not_less_than_1_and_b_must_not_greater_than_2(a, b):
return a >= 1 and b <= 2
.. note::
If the check method raises any exception, this rule will be
regarded as `not matched`. This design purpose is to ease
the rules like `matching a string that can be converted into
int`.
:return: The decorator.
"""
def outer(method):
@wraps(method)
def inner(*args, **kwargs):
try:
return method(*args, **kwargs)
except Exception:
return False
setattr(inner, 'description', description)
self.check_classes.append(inner)
return method
return outer
def do_run(self):
# First, load the data, and format the error message if schema
# is not matched.
try:
loaded_data = list(CsvSchema.LoadCSV(self.schema, self.csvdata))
except KeyError, ex:
raise ScorerFailure(
brief=lazy_gettext('CSV data does not match schema.'),
detail=[ex.args[0]])
except ValueError, ex:
raise ScorerFailure(
brief=lazy_gettext('CSV data does not match schema.'),
detail=[ex.args[0]])
def addSkip(self, test, reason):
super(UnitTestScorerDetailResult, self).addSkip(test, reason)
self.details.append(
lazy_gettext('SKIP: %(test)s: %(reason)s.',
test=self.getDescription(test),
reason=reason))
def __init__(self, **kwargs):
super(ExtractFileFailure, self).__init__(
lazy_gettext("Couldn't extract your archive file."), **kwargs)
def addError(self, test, err):
super(UnitTestScorerDetailResult, self).addError(test, err)
self.details.append(
lazy_gettext('ERROR: %(test)s.\n%(error)s',
test=self.getDescription(test),
error=self._exc_info_to_string(err, test)))
def __init__(self, schema):
super(ObjSchemaScorer,
self).__init__(lazy_gettext('Object Structure Scorer'))
self.schema = schema
def addSuccess(self, test):
super(UnitTestScorerDetailResult, self).addSuccess(test)
self.details.append(lazy_gettext(
'PASSED: %(test)s.',
test=self.getDescription(test)
))
def api_handin_report(uuid):
"""Store the final score and detailed reports of given submission.
This view will compare `uuid` in POST object to the `uuid` argument.
If they are not equal, the operation will be rejected, since it is
likely to be an attack.
If the submission state is neither `Running` nor `Pending`, the operation
will be rejected, since it is likely to be a programmatic bug.
If the reported score is 0.0, but the state is `Accepted`, then it
will be modified to `Rejected`, since it is wired for a zero-score
submission to be `Accepted`.
If the reported brief result message is empty, it will be set to
a translated version of `"Your submission is accepted."` or
`"Your submission is rejected."`, depending on the reported state.
The :class:`~railgun.website.models.FinalScore` table records will
also be updated in this view.
:route: /api/handin/report/<uuid>/
:payload: A serialized :class:`~railgun.common.hw.HwScore` object.
:param uuid: The uuid of submission.
:type uuid: :class:`str`
:return: ``OK`` if succeeded, error messages otherwise.
"""
obj = request.payload
# check uuid, so that we can prevent replay attack
if obj['uuid'] != uuid:
return 'uuid mismatch, do not attack'
# construct HwScore object from payload
try:
score = HwScore.from_plain(obj)
except Exception:
return 'not valid score object'
# load the handin object, and report error if not exist
handin = Handin.query.filter(Handin.uuid == uuid).first()
if not handin:
return 'requested handin not found'
# if handin.state not in ['Running', 'Pending'], it must already have a
# score. reject the API call.
if handin.state != 'Running' and handin.state != 'Pending':
return 'score already reported'
# Special hack: unittest will catch all exceptions.
#
# Such submissions may result 0.0 base score but marked as 'Accepted'.
# I decide to treat these submissions 'Rejected', because no one
# would accept a totally bad submission.
handin.score = score.get_score()
if handin.score < 1e-5 and score.accepted:
score.accepted = False
score.result = lazy_gettext('No test passed, submission rejected.')
# update result of handin
handin.state = 'Accepted' if score.accepted else 'Rejected'
if score.accepted:
handin.result = lazy_gettext('Your submission is accepted.')
elif unicode(score.result):
handin.result = score.result
else:
handin.result = lazy_gettext('Your submission is rejected.')
handin.compile_error = score.compile_error
handin.partials = score.partials
# update hwscore table and set the final score of this homework
if handin.is_accepted():
final_score = handin.score * handin.scale
hwscore = (FinalScore.query.filter(
FinalScore.hwid == handin.hwid).filter(
FinalScore.user_id == handin.user_id)).first()
if not hwscore:
hwscore = FinalScore(user_id=handin.user_id,
hwid=handin.hwid,
score=final_score)
db.session.add(hwscore)
elif final_score > hwscore.score:
hwscore.score = final_score
try:
db.session.commit()
except Exception:
app.logger.exception('Cannot update result of submission(%s).' % uuid)
return 'update database failed'
return 'OK'
def __init__(self, **kwargs):
super(RunnerPermissionError, self).__init__(
lazy_gettext('File permissions of the runner is wrong.'), **kwargs)