def create_build(c, job_name, source, commit, message):
build_number = get_next_autoincrementing_value(c, "dockergrader_last_build_number")
build_name = "%s-build-%d" % (job_name, build_number)
c.execute('''INSERT INTO builds (build_name, source, `commit`, message, job, status, score,
started, updated, log) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
[build_name, source, commit, message, job_name, QUEUED, 0.0, now_str(),
now_str(), None])
return build_name
def create_build(c, job_name, source, commit, message):
build_number = get_next_autoincrementing_value(c, "dockergrader_last_build_number")
build_name = "%s-build-%d" % (job_name, build_number)
c.execute('''INSERT INTO builds (build_name, source, `commit`, message, job, status, score,
started, updated, log) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
[build_name, source, commit, message, job_name, QUEUED, 0.0, now_str(),
now_str(), None])
return build_name
def rate_limit_fail_build(build_name):
assert MAX_JOBS_ALLOWED is not None
message = "Cannot have more than {} builds in progress or queued.".format(
MAX_JOBS_ALLOWED)
with DbCursor() as c:
c.execute(
'''UPDATE builds SET status = ?, updated = ?, log = ?
WHERE build_name = ?''',
[FAILED, now_str(), message, build_name])
def create(self, c, operation, payload):
"""
Creates a new queue job and returns an opaque object representing the job. The new job will
be part of the transaction, so if the transaction is rolled back, this job will disappear
too.
If the transaction is successful, you should pass the opaque object returned by this method
to enqueue(), so the queue runner can process it. Otherwise, it will be processed during
the next re-start of the server daemon when uncompleted jobs are retried.
"""
transaction_id = self.get_transaction_id(c)
c.execute(
"""INSERT INTO %s (id, operation, payload, updated, completed)
VALUES (?, ?, ?, ?, ?)"""
% self.database_table,
[transaction_id, operation, self.serialize_arguments(payload), now_str(), 0],
)
return (transaction_id, operation, payload)
def create(self, c, operation, payload):
"""
Creates a new queue job and returns an opaque object representing the job. The new job will
be part of the transaction, so if the transaction is rolled back, this job will disappear
too.
If the transaction is successful, you should pass the opaque object returned by this method
to enqueue(), so the queue runner can process it. Otherwise, it will be processed during
the next re-start of the server daemon when uncompleted jobs are retried.
"""
transaction_id = self.get_transaction_id(c)
c.execute(
'''INSERT INTO %s (id, operation, payload, updated, completed)
VALUES (?, ?, ?, ?, ?)''' % self.database_table, [
transaction_id, operation,
self.serialize_arguments(payload),
now_str(), 0
])
return (transaction_id, operation, payload)
def assign_grade_batch(c, users, assignment, score, slipunits, transaction_name, description,
source, manual=False, dont_lower=False):
"""
Assigns a new grade to one or more students. Also supports assigning slip units. You can use
the special value `None` for score and/or slipunits to use the current value. If the dont_lower
flag is True, then anybody in `users` who currently has a higher grade will be removed from
the operation (and slip days will not be adjusted either).
Returns a list of user ids whose grades were affected (will always be a subset of users).
"""
if assignment not in get_assignment_name_set():
raise ValueError("Assignment %s is not known" % assignment)
if not users:
return []
if score is None and slipunits is None:
return []
timestamp = now_str()
if dont_lower:
if score is None:
# It makes no sense to do this.
raise ValueError("You can not use both dont_lower=True and have a score of None, if " +
"slipunits is not None.")
# This comparison (old score vs new score) MUST be done in Sqlite in order for the result
# to be correct. Sqlite will round the floating point number in the same way it did when
# the original result was inserted, and the two values will be equal. Converting this to a
# float in another language may produce undesirable effects.
c.execute('''SELECT user FROM grades
WHERE assignment = ? AND score >= ? AND user IN (%s)''' %
(','.join(['?'] * len(users))), [assignment, score] + users)
users = list(set(users) - {user for user, in c.fetchall()})
if not users:
return []
c.execute('''SELECT users.id FROM grades LEFT JOIN users
ON grades.user = users.id WHERE grades.assignment = ? AND users.id IN (%s)''' %
(','.join(["?"] * len(users))), [assignment] + users)
for user in list(set(users) - {user for user, in c.fetchall()}):
# Insert dummy values first, and we will update them later
c.execute('''INSERT INTO grades (user, assignment) VALUES (?, ?)''',
[user, assignment])
c.execute('''UPDATE grades SET updated = ?, manual = ?
WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))),
[timestamp, int(manual), assignment] + users)
if score is not None:
c.execute('''UPDATE grades SET score = ?
WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))),
[score, assignment] + users)
if slipunits is not None:
c.execute('''UPDATE grades SET slipunits = ?
WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))),
[slipunits, assignment] + users)
c.execute('''INSERT INTO gradeslog (transaction_name, description, source, updated, user,
assignment, score, slipunits)
VALUES %s''' % (','.join(['(?,?,?,?,?,?,?,?)'] * len(users))),
[field for entry in [[transaction_name, description, source, timestamp, user,
assignment, score, slipunits]
for user in users] for field in entry])
return users
def test_time_functions(self):
timestamp_str = now_str()
timestamp_obj = parse_time(timestamp_str)
self.assertEqual(timestamp_str, format_time(timestamp_obj))
def _process_job(self, job):
build_name = job.build_name
with self.lock:
self.status = build_name
self.updated = now()
# Mark the job as In Progress
while True:
try:
with DbCursor() as c:
c.execute('''SELECT source, `commit`, message, job, started FROM builds
WHERE build_name = ? AND status = ? LIMIT 1''',
[build_name, QUEUED])
row = c.fetchone()
if row is None:
self._log("Build %s was missing from the database. Skipping." % build_name)
return
source, commit, message, job_name, started = row
owners = get_repo_owners(c, source)
owner_emails = {owner: email for owner, (_, _, _, _, _, email)
in get_users_by_ids(c, owners).items()}
c.execute("UPDATE builds SET status = ?, updated = ? WHERE build_name = ?",
[IN_PROGRESS, now_str(), build_name])
break
except apsw.Error:
self._log("Exception raised while setting status to IN_PROGRESS. Retrying...",
exc=True)
logging.exception("Failed to retrieve next dockergrader job")
self._log("Started building %s" % build_name)
try:
# if the job doesn't exist for some reason, the resulting TypeError will be caught
# and logged
assignment = get_assignment_by_name(job_name)
due_date = assignment.due_date
job_handler = get_job(job_name)
log, score = job_handler(source, commit)
# Ignore any special encoding inside the log, and just treat it as a bytes
log = buffer(log)
min_score, max_score = assignment.min_score, assignment.max_score
full_score = assignment.full_score
if score < min_score or score > max_score:
raise ValueError("A score of %s is not in the acceptable range of %f to %f" %
(str(score), min_score, max_score))
except JobFailedError as e:
self._log("Failed %s with JobFailedError" % build_name, exc=True)
with DbCursor() as c:
c.execute('''UPDATE builds SET status = ?, updated = ?, log = ?
WHERE build_name = ?''', [FAILED, now_str(), str(e), build_name])
if config.mailer_enabled:
try:
for owner in owners:
email = owner_emails.get(owner)
if not email:
continue
subject = "%s failed to complete" % build_name
send_template("build_failed", email, subject, build_name=build_name,
job_name=job_name, source=source, commit=commit,
message=message, error_message=str(e))
except Exception:
self._log("Exception raised while reporting JobFailedError", exc=True)
logging.exception("Exception raised while reporting JobFailedError")
else:
self._log("JobFailedError successfully reported via email")
return
except Exception as e:
self._log("Exception raised while building %s" % build_name, exc=True)
logging.exception("Internal error within build %s" % build_name)
with DbCursor() as c:
c.execute('''UPDATE builds SET status = ?, updated = ?, log = ?
WHERE build_name = ?''',
[FAILED, now_str(), "Build failed due to an internal error.", build_name])
return
self._log("Autograder build %s complete (score: %s)" % (build_name, str(score)))
while True:
try:
with DbCursor() as c:
c.execute('''UPDATE builds SET status = ?, score = ?, updated = ?,
log = ? WHERE build_name = ?''',
[SUCCESS, score, now_str(), log, build_name])
slipunits = slip_units(due_date, started)
affected_users = assign_grade_batch(c, owners, job_name, float(score),
slipunits, build_name, "Automatic build.",
"autograder", dont_lower=True)
break
except apsw.Error:
self._log("Exception raised while assigning grades", exc=True)
logging.exception("Failed to update build %s after build completed" % build_name)
return
if config.mailer_enabled:
try:
for owner in owners:
email = owner_emails.get(owner)
if not email:
continue
subject = "%s complete - score %s / %s" % (build_name, str(score),
str(full_score))
if owner not in affected_users:
subject += " (no effect on grade)"
else:
if slipunits == 1:
subject += " (1 %s used)" % config.slip_unit_name_singular
elif slipunits > 0:
subject += " (%s slip %s used)" % (str(slipunits),
config.slip_unit_name_plural)
send_template("build_finished", email, subject, build_name=build_name,
job_name=job_name, score=score, full_score=str(full_score),
slipunits=slipunits, log=log, source=source, commit=commit,
message=message, affected=(owner in affected_users))
except Exception:
self._log("Exception raised while reporting grade", exc=True)
logging.exception("Exception raised while reporting grade")
else:
self._log("Grade successfully reported via email")
def test_time_functions(self):
timestamp_str = now_str()
timestamp_obj = parse_time(timestamp_str)
self.assertEqual(timestamp_str, format_time(timestamp_obj))
def assign_grade_batch(c, users, assignment, score, slipunits, transaction_name, description,
source, manual=False, dont_lower=False):
"""
Assigns a new grade to one or more students. Also supports assigning slip units. You can use
the special value `None` for score and/or slipunits to use the current value. If the dont_lower
flag is True, then anybody in `users` who currently has a higher grade will be removed from
the operation (and slip days will not be adjusted either).
Returns a list of user ids whose grades were affected (will always be a subset of users).
"""
if assignment not in get_assignment_name_set():
raise ValueError("Assignment %s is not known" % assignment)
if not users:
return []
if score is None and slipunits is None:
return []
timestamp = now_str()
if dont_lower:
if score is None:
# It makes no sense to do this.
raise ValueError("You can not use both dont_lower=True and have a score of None, if " +
"slipunits is not None.")
# This comparison (old score vs new score) MUST be done in Sqlite in order for the result
# to be correct. Sqlite will round the floating point number in the same way it did when
# the original result was inserted, and the two values will be equal. Converting this to a
# float in another language may produce undesirable effects.
c.execute('''SELECT user FROM grades
WHERE assignment = ? AND score >= ? AND user IN (%s)''' %
(','.join(['?'] * len(users))), [assignment, score] + users)
users = list(set(users) - {user for user, in c.fetchall()})
if not users:
return []
c.execute('''SELECT users.id FROM grades LEFT JOIN users
ON grades.user = users.id WHERE grades.assignment = ? AND users.id IN (%s)''' %
(','.join(["?"] * len(users))), [assignment] + users)
for user in list(set(users) - {user for user, in c.fetchall()}):
# Insert dummy values first, and we will update them later
c.execute('''INSERT INTO grades (user, assignment) VALUES (?, ?)''',
[user, assignment])
c.execute('''UPDATE grades SET updated = ?, manual = ?
WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))),
[timestamp, int(manual), assignment] + users)
if score is not None:
c.execute('''UPDATE grades SET score = ?
WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))),
[score, assignment] + users)
if slipunits is not None:
c.execute('''UPDATE grades SET slipunits = ?
WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))),
[slipunits, assignment] + users)
c.execute('''INSERT INTO gradeslog (transaction_name, description, source, updated, user,
assignment, score, slipunits)
VALUES %s''' % (','.join(['(?,?,?,?,?,?,?,?)'] * len(users))),
[field for entry in [[transaction_name, description, source, timestamp, user,
assignment, score, slipunits]
for user in users] for field in entry])
return users
