def create_build(c, job_name, source, commit, message): build_number = get_next_autoincrementing_value(c, "dockergrader_last_build_number") build_name = "%s-build-%d" % (job_name, build_number) c.execute('''INSERT INTO builds (build_name, source, `commit`, message, job, status, score, started, updated, log) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', [build_name, source, commit, message, job_name, QUEUED, 0.0, now_str(), now_str(), None]) return build_name
def rate_limit_fail_build(build_name): assert MAX_JOBS_ALLOWED is not None message = "Cannot have more than {} builds in progress or queued.".format( MAX_JOBS_ALLOWED) with DbCursor() as c: c.execute( '''UPDATE builds SET status = ?, updated = ?, log = ? WHERE build_name = ?''', [FAILED, now_str(), message, build_name])
def create(self, c, operation, payload): """ Creates a new queue job and returns an opaque object representing the job. The new job will be part of the transaction, so if the transaction is rolled back, this job will disappear too. If the transaction is successful, you should pass the opaque object returned by this method to enqueue(), so the queue runner can process it. Otherwise, it will be processed during the next re-start of the server daemon when uncompleted jobs are retried. """ transaction_id = self.get_transaction_id(c) c.execute( """INSERT INTO %s (id, operation, payload, updated, completed) VALUES (?, ?, ?, ?, ?)""" % self.database_table, [transaction_id, operation, self.serialize_arguments(payload), now_str(), 0], ) return (transaction_id, operation, payload)
def create(self, c, operation, payload): """ Creates a new queue job and returns an opaque object representing the job. The new job will be part of the transaction, so if the transaction is rolled back, this job will disappear too. If the transaction is successful, you should pass the opaque object returned by this method to enqueue(), so the queue runner can process it. Otherwise, it will be processed during the next re-start of the server daemon when uncompleted jobs are retried. """ transaction_id = self.get_transaction_id(c) c.execute( '''INSERT INTO %s (id, operation, payload, updated, completed) VALUES (?, ?, ?, ?, ?)''' % self.database_table, [ transaction_id, operation, self.serialize_arguments(payload), now_str(), 0 ]) return (transaction_id, operation, payload)
def assign_grade_batch(c, users, assignment, score, slipunits, transaction_name, description, source, manual=False, dont_lower=False): """ Assigns a new grade to one or more students. Also supports assigning slip units. You can use the special value `None` for score and/or slipunits to use the current value. If the dont_lower flag is True, then anybody in `users` who currently has a higher grade will be removed from the operation (and slip days will not be adjusted either). Returns a list of user ids whose grades were affected (will always be a subset of users). """ if assignment not in get_assignment_name_set(): raise ValueError("Assignment %s is not known" % assignment) if not users: return [] if score is None and slipunits is None: return [] timestamp = now_str() if dont_lower: if score is None: # It makes no sense to do this. raise ValueError("You can not use both dont_lower=True and have a score of None, if " + "slipunits is not None.") # This comparison (old score vs new score) MUST be done in Sqlite in order for the result # to be correct. Sqlite will round the floating point number in the same way it did when # the original result was inserted, and the two values will be equal. Converting this to a # float in another language may produce undesirable effects. c.execute('''SELECT user FROM grades WHERE assignment = ? AND score >= ? AND user IN (%s)''' % (','.join(['?'] * len(users))), [assignment, score] + users) users = list(set(users) - {user for user, in c.fetchall()}) if not users: return [] c.execute('''SELECT users.id FROM grades LEFT JOIN users ON grades.user = users.id WHERE grades.assignment = ? AND users.id IN (%s)''' % (','.join(["?"] * len(users))), [assignment] + users) for user in list(set(users) - {user for user, in c.fetchall()}): # Insert dummy values first, and we will update them later c.execute('''INSERT INTO grades (user, assignment) VALUES (?, ?)''', [user, assignment]) c.execute('''UPDATE grades SET updated = ?, manual = ? WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))), [timestamp, int(manual), assignment] + users) if score is not None: c.execute('''UPDATE grades SET score = ? WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))), [score, assignment] + users) if slipunits is not None: c.execute('''UPDATE grades SET slipunits = ? WHERE assignment = ? AND user IN (%s)''' % (','.join(['?'] * len(users))), [slipunits, assignment] + users) c.execute('''INSERT INTO gradeslog (transaction_name, description, source, updated, user, assignment, score, slipunits) VALUES %s''' % (','.join(['(?,?,?,?,?,?,?,?)'] * len(users))), [field for entry in [[transaction_name, description, source, timestamp, user, assignment, score, slipunits] for user in users] for field in entry]) return users
def test_time_functions(self): timestamp_str = now_str() timestamp_obj = parse_time(timestamp_str) self.assertEqual(timestamp_str, format_time(timestamp_obj))
def _process_job(self, job): build_name = job.build_name with self.lock: self.status = build_name self.updated = now() # Mark the job as In Progress while True: try: with DbCursor() as c: c.execute('''SELECT source, `commit`, message, job, started FROM builds WHERE build_name = ? AND status = ? LIMIT 1''', [build_name, QUEUED]) row = c.fetchone() if row is None: self._log("Build %s was missing from the database. Skipping." % build_name) return source, commit, message, job_name, started = row owners = get_repo_owners(c, source) owner_emails = {owner: email for owner, (_, _, _, _, _, email) in get_users_by_ids(c, owners).items()} c.execute("UPDATE builds SET status = ?, updated = ? WHERE build_name = ?", [IN_PROGRESS, now_str(), build_name]) break except apsw.Error: self._log("Exception raised while setting status to IN_PROGRESS. Retrying...", exc=True) logging.exception("Failed to retrieve next dockergrader job") self._log("Started building %s" % build_name) try: # if the job doesn't exist for some reason, the resulting TypeError will be caught # and logged assignment = get_assignment_by_name(job_name) due_date = assignment.due_date job_handler = get_job(job_name) log, score = job_handler(source, commit) # Ignore any special encoding inside the log, and just treat it as a bytes log = buffer(log) min_score, max_score = assignment.min_score, assignment.max_score full_score = assignment.full_score if score < min_score or score > max_score: raise ValueError("A score of %s is not in the acceptable range of %f to %f" % (str(score), min_score, max_score)) except JobFailedError as e: self._log("Failed %s with JobFailedError" % build_name, exc=True) with DbCursor() as c: c.execute('''UPDATE builds SET status = ?, updated = ?, log = ? WHERE build_name = ?''', [FAILED, now_str(), str(e), build_name]) if config.mailer_enabled: try: for owner in owners: email = owner_emails.get(owner) if not email: continue subject = "%s failed to complete" % build_name send_template("build_failed", email, subject, build_name=build_name, job_name=job_name, source=source, commit=commit, message=message, error_message=str(e)) except Exception: self._log("Exception raised while reporting JobFailedError", exc=True) logging.exception("Exception raised while reporting JobFailedError") else: self._log("JobFailedError successfully reported via email") return except Exception as e: self._log("Exception raised while building %s" % build_name, exc=True) logging.exception("Internal error within build %s" % build_name) with DbCursor() as c: c.execute('''UPDATE builds SET status = ?, updated = ?, log = ? WHERE build_name = ?''', [FAILED, now_str(), "Build failed due to an internal error.", build_name]) return self._log("Autograder build %s complete (score: %s)" % (build_name, str(score))) while True: try: with DbCursor() as c: c.execute('''UPDATE builds SET status = ?, score = ?, updated = ?, log = ? WHERE build_name = ?''', [SUCCESS, score, now_str(), log, build_name]) slipunits = slip_units(due_date, started) affected_users = assign_grade_batch(c, owners, job_name, float(score), slipunits, build_name, "Automatic build.", "autograder", dont_lower=True) break except apsw.Error: self._log("Exception raised while assigning grades", exc=True) logging.exception("Failed to update build %s after build completed" % build_name) return if config.mailer_enabled: try: for owner in owners: email = owner_emails.get(owner) if not email: continue subject = "%s complete - score %s / %s" % (build_name, str(score), str(full_score)) if owner not in affected_users: subject += " (no effect on grade)" else: if slipunits == 1: subject += " (1 %s used)" % config.slip_unit_name_singular elif slipunits > 0: subject += " (%s slip %s used)" % (str(slipunits), config.slip_unit_name_plural) send_template("build_finished", email, subject, build_name=build_name, job_name=job_name, score=score, full_score=str(full_score), slipunits=slipunits, log=log, source=source, commit=commit, message=message, affected=(owner in affected_users)) except Exception: self._log("Exception raised while reporting grade", exc=True) logging.exception("Exception raised while reporting grade") else: self._log("Grade successfully reported via email")