def score_from_csv(assign_id, rows, kind='total', invalid=None, message=None):
"""
Job for uploading Scores.
@param ``rows`` should be a list of records (mappings),
with labels `email` and `score`
"""
log = jobs.get_job_logger()
current_user = jobs.get_current_job().user
assign = Assignment.query.get(assign_id)
message = message or '{} score for {}'.format(kind.title(), assign.display_name)
def log_err(msg):
log.info('\t! {}'.format(msg))
log.info("Uploading scores for {}:\n".format(assign.display_name))
if invalid:
log_err('skipping {} invalid entries on lines:'.format(len(invalid)))
for line in invalid:
log_err('\t{}'.format(line))
log.info('')
success, total = 0, len(rows)
for i, row in enumerate(rows, start=1):
try:
email, score = row['email'], row['score']
user = User.query.filter_by(email=email).one()
backup = Backup.query.filter_by(assignment=assign, submitter=user, submit=True).first()
if not backup:
backup = Backup.create(submitter=user, assignment=assign, submit=True)
uploaded_score = Score(grader=current_user, assignment=assign,
backup=backup, user=user, score=score, kind=kind, message=message)
db.session.add(uploaded_score)
uploaded_score.archive_duplicates()
except SQLAlchemyError:
print_exc()
log_err('error: user with email `{}` does not exist'.format(email))
else:
success += 1
if i % 100 == 0:
log.info('\nUploaded {}/{} Scores\n'.format(i, total))
db.session.commit()
log.info('\nSuccessfully uploaded {} "{}" scores (with {} errors)'.format(success, kind, total - success))
return '/admin/course/{cid}/assignments/{aid}/scores'.format(
cid=jobs.get_current_job().course_id, aid=assign_id)
def export_grades():
logger = jobs.get_job_logger()
current_user = jobs.get_current_job().user
course = Course.query.get(jobs.get_current_job().course_id)
assignments = course.assignments
students = (Enrollment.query.options(db.joinedload('user')).filter(
Enrollment.role == STUDENT_ROLE, Enrollment.course == course).all())
headers, assignments = get_headers(assignments)
logger.info("Using these headers:")
for header in headers:
logger.info('\t' + header)
logger.info('')
total_students = len(students)
users = [student.user for student in students]
user_ids = [user.id for user in users]
all_scores = collect_all_scores(assignments, user_ids)
with io.StringIO() as f:
writer = csv.writer(f)
writer.writerow(headers) # write headers
for i, student in enumerate(students, start=1):
row = export_student_grades(student, assignments, all_scores)
writer.writerow(row)
if i % 50 == 0:
logger.info('Exported {}/{}'.format(i, total_students))
f.seek(0)
created_time = local_time(dt.datetime.now(),
course,
fmt='%b-%-d %Y at %I-%M%p')
csv_filename = '{course_name} Grades ({date}).csv'.format(
course_name=course.display_name, date=created_time)
# convert to bytes for csv upload
csv_bytes = io.BytesIO(bytearray(f.read(), 'utf-8'))
upload = ExternalFile.upload(csv_bytes,
user_id=current_user.id,
name=csv_filename,
course_id=course.id,
prefix='jobs/exports/{}/'.format(
course.offering))
logger.info('\nDone!\n')
logger.info("Saved as: {0}".format(upload.object_name))
return "/files/{0}".format(encode_id(upload.id))
def autograde_assignment(assignment_id):
"""Autograde all enrolled students for this assignment.
We set up a state machine for each backup to check its progress through
the autograder. If any step takes too long, we'll retry autograding that
backup. Ultimately, a backup is considered done when we confirm that
we've received a new score, or if we have reached the retry limit.
"""
logger = jobs.get_job_logger()
assignment = Assignment.query.get(assignment_id)
course_submissions = assignment.course_submissions(include_empty=False)
backup_ids = set(fs['backup']['id'] for fs in course_submissions if fs['backup'])
try:
autograde_backups(assignment, jobs.get_current_job().user_id, backup_ids, logger)
except ValueError:
logger.info('Could not autograde backups - Please add an autograding key.')
return
return '/admin/course/{cid}/assignments/{aid}/scores'.format(
cid=jobs.get_current_job().course_id, aid=assignment.id)
def email_scores(assignment_id, score_tags, subject, body,
reply_to=None, dry_run=False):
log = jobs.get_job_logger()
job_creator = jobs.get_current_job().user
assign = Assignment.query.get(assignment_id)
students = [e.user for e in (Enrollment.query
.options(db.joinedload('user'))
.filter(Enrollment.role == STUDENT_ROLE,
Enrollment.course == assign.course)
.all())]
email_counter = 0
seen_ids = set()
for student in students:
if student.id in seen_ids:
continue
user_ids = assign.active_user_ids(student.id)
seen_ids |= user_ids
scores = [s for s in assign.scores(user_ids) if s.kind in score_tags]
if scores:
users = User.query.filter(User.id.in_(user_ids))
primary, cc = users[0].email, [u.email for u in users[1:]]
if dry_run:
primary, cc = job_creator.email, []
result = send_email(primary,
subject, body,
cc=cc,
template='email/scores.html',
title=subject,
from_name=assign.course.display_name,
scores=scores,
reply_to=reply_to,
link_text="View on okpy.org",
link="https://okpy.org/" + assign.name, # Don't have url_for
assignment=assign.display_name)
if result:
log.info("Sent to {}".format(', '.join([primary] + cc)))
email_counter += 1
# Send a few emails in dry run mode.
if dry_run and email_counter >= 2:
message = "Run with dry run mode"
log.info(message)
return message
message = "Sent {} emails".format(email_counter)
log.info(message)
return message
def autograde_assignment(assignment_id):
"""Autograde all enrolled students for this assignment.
We set up a state machine for each backup to check its progress through
the autograder. If any step takes too long, we'll retry autograding that
backup. Ultimately, a backup is considered done when we confirm that
we've received a new score, or if we have reached the retry limit.
"""
logger = jobs.get_job_logger()
assignment = Assignment.query.get(assignment_id)
course_submissions = assignment.course_submissions(include_empty=False)
backup_ids = set(fs['backup']['id'] for fs in course_submissions
if fs['backup'])
try:
autograde_backups(assignment,
jobs.get_current_job().user_id, backup_ids, logger)
except ValueError:
logger.info(
'Could not autograde backups - Please add an autograding key.')
return
return '/admin/course/{cid}/assignments/{aid}/scores'.format(
cid=jobs.get_current_job().course_id, aid=assignment.id)
def export_grades():
logger = jobs.get_job_logger()
current_user = jobs.get_current_job().user
course = Course.query.get(jobs.get_current_job().course_id)
assignments = course.assignments
students = (Enrollment.query
.options(db.joinedload('user'))
.filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course)
.all())
headers, assignments = get_headers(assignments)
logger.info("Using these headers:")
for header in headers:
logger.info('\t' + header)
logger.info('')
total_students = len(students)
with io.StringIO() as f:
writer = csv.writer(f)
writer.writerow(headers) # write headers
for i, student in enumerate(students, start=1):
row = export_student_grades(student, assignments)
writer.writerow(row)
if i % 50 == 0:
logger.info('Exported {}/{}'.format(i, total_students))
f.seek(0)
created_time = local_time(dt.datetime.now(), course, fmt='%b-%-d %Y at %I-%M%p')
csv_filename = '{course_name} Grades ({date}).csv'.format(
course_name=course.display_name, date=created_time)
# convert to bytes for csv upload
csv_bytes = io.BytesIO(bytearray(f.read(), 'utf-8'))
upload = ExternalFile.upload(csv_bytes, user_id=current_user.id, name=csv_filename,
course_id=course.id,
prefix='jobs/exports/{}/'.format(course.offering))
logger.info('\nDone!\n')
logger.info("Saved as: {0}".format(upload.object_name))
return "/files/{0}".format(encode_id(upload.id))
def assign_scores(assign_id, score, kind, message, deadline,
include_backups=True):
logger = jobs.get_job_logger()
current_user = jobs.get_current_job().user
assignment = Assignment.query.get(assign_id)
students = [e.user_id for e in assignment.course.get_students()]
submission_time = server_time_obj(deadline, assignment.course)
# Find all submissions (or backups) before the deadline
backups = Backup.query.filter(
Backup.assignment_id == assign_id,
or_(Backup.created <= deadline, Backup.custom_submission_time <= deadline)
).order_by(Backup.created.desc()).group_by(Backup.submitter_id)
if not include_backups:
backups = backups.filter(Backup.submit == True)
all_backups = backups.all()
if not all_backups:
logger.info("No submissions were found with a deadline of {}."
.format(deadline))
return "No Scores Created"
total_count = len(all_backups)
logger.info("Found {} eligible submissions...".format(total_count))
score_counter, seen = 0, set()
for back in all_backups:
if back.creator in seen:
score_counter += 1
continue
new_score = Score(score=score, kind=kind, message=message,
user_id=back.submitter_id,
assignment=assignment, backup=back,
grader=current_user)
db.session.add(new_score)
new_score.archive_duplicates()
db.session.commit()
score_counter += 1
if score_counter % 5 == 0:
logger.info("Scored {} of {}".format(score_counter, total_count))
seen |= back.owners()
result = "Left {} '{}' scores of {}".format(score_counter, kind.title(), score)
logger.info(result)
return result
def autograde_assignment(assignment_id):
"""Autograde all enrolled students for this assignment.
We set up a state machine for each backup to check its progress through
the autograder. If any step takes too long, we'll retry autograding that
backup. Ultimately, a backup is considered done when we confirm that
we've received a new score, or if we have reached the retry limit.
"""
logger = jobs.get_job_logger()
assignment = Assignment.query.get(assignment_id)
course_submissions = assignment.course_submissions(include_empty=False)
backup_ids = set(fs['backup']['id'] for fs in course_submissions
if fs['backup'])
return autograde_backups(assignment,
jobs.get_current_job().user_id, backup_ids,
logger)
def export_assignment(assignment_id, anonymized):
""" Generate a zip file of submissions from enrolled students.
Final Submission: One submission per student/group
Zip Strucutre: cal-cs61a../[email protected]@b.com/abc12d/hog.py
Anonymized: Submission without identifying info
Zip Strucutre: cal-cs61a../{hash}/hog.py
"""
logger = jobs.get_job_logger()
assignment = Assignment.query.get(assignment_id)
requesting_user = jobs.get_current_job().user
if not assignment:
logger.warning("No assignment found")
raise Exception("No Assignment")
if not Assignment.can(assignment, requesting_user, "download"):
raise Exception("{} does not have enough permission"
.format(requesting_user.email))
if anonymized:
logger.info("Starting anonymized submission export")
else:
logger.info("Starting final submission export")
course = assignment.course
with io.BytesIO() as bio:
# Get a handle to the in-memory zip in append mode
with zipfile.ZipFile(bio, "w", zipfile.ZIP_DEFLATED, False) as zf:
zf.external_attr = 0o655 << 16
export_loop(bio, zf, logger, assignment, anonymized)
created_time = local_time(dt.datetime.now(), course, fmt='%m-%d-%I-%M-%p')
zip_name = '{}_{}.zip'.format(assignment.name.replace('/', '-'), created_time)
bio.seek(0)
# Close zf handle to finish writing zipfile
logger.info("Uploading...")
upload = ExternalFile.upload(bio, user_id=requesting_user.id, name=zip_name,
course_id=course.id,
prefix='jobs/exports/{}/'.format(course.offering))
logger.info("Saved as: {0}".format(upload.object_name))
msg = "/files/{0}".format(encode_id(upload.id))
return msg
def autograde_assignment(assignment_id):
"""Autograde all enrolled students for this assignment.
We set up a state machine for each backup to check its progress through
the autograder. If any step takes too long, we'll retry autograding that
backup. Ultimately, a backup is considered done when we confirm that
we've received a new score, or if we have reached the retry limit.
"""
logger = jobs.get_job_logger()
assignment = Assignment.query.get(assignment_id)
course_submissions = assignment.course_submissions(include_empty=False)
backup_ids = set(fs['backup']['id'] for fs in course_submissions
if fs['backup'])
token = create_autograder_token(jobs.get_current_job().user_id)
# start by sending a batch of all backups
start_time = time.time()
job_ids = send_batch(token, assignment, backup_ids)
tasks = [
GradingTask(
status=GradingStatus.QUEUED,
backup_id=backup_id,
job_id=job_id,
retries=0,
) for backup_id, job_id in job_ids.items()
]
num_tasks = len(tasks)
def retry_task(task):
if task.retries >= MAX_RETRIES:
logger.error(
'Did not receive a score for backup {} after {} retries'.
format(utils.encode_id(task.backup_id), MAX_RETRIES))
task.set_status(GradingStatus.FAILED)
else:
task.set_status(GradingStatus.QUEUED)
task.job_id = autograde_backup(token, assignment, task.backup_id)
task.retries += 1
while True:
time.sleep(POLL_INTERVAL)
results = check_job_results([task.job_id for task in tasks])
graded = len([
task for task in tasks
if task.status in (GradingStatus.DONE, GradingStatus.FAILED)
])
logger.info('Graded {:>4}/{} ({:>5.1f}%)'.format(
graded, num_tasks, 100 * graded / num_tasks))
if graded == num_tasks:
break
for task in tasks:
hashid = utils.encode_id(task.backup_id)
if task.status == GradingStatus.QUEUED:
result = results[task.job_id]
if not result:
logger.warning(
'Autograder job {} disappeared, retrying'.format(
task.job_id))
retry_task(task)
elif result['status'] != 'queued':
logger.debug(
'Autograder job {} for backup {} started'.format(
task.job_id, hashid))
task.set_status(GradingStatus.RUNNING)
elif task.expired(QUEUED_TIMEOUT):
logger.warning(
'Autograder job {} queued longer than {} seconds, retrying'
.format(task.job_id, QUEUED_TIMEOUT))
retry_task(task)
elif task.status == GradingStatus.RUNNING:
result = results[task.job_id]
if not result:
logger.warning(
'Autograder job {} disappeared, retrying'.format(
task.job_id))
retry_task(task)
elif result['status'] == 'finished':
logger.debug(
'Autograder job {} for backup {} finished'.format(
task.job_id, hashid))
task.set_status(GradingStatus.WAITING)
elif result['status'] == 'failed':
logger.warning('Autograder job {} failed, retrying'.format(
task.job_id))
retry_task(task)
elif task.expired(RUNNING_TIMEOUT):
logger.warning(
'Autograder job {} running longer than {} seconds, retrying'
.format(task.job_id, RUNNING_TIMEOUT))
retry_task(task)
elif task.status == GradingStatus.WAITING:
score = Score.query.filter(
Score.backup_id == task.backup_id, Score.archived == False,
Score.created >
datetime.datetime.fromtimestamp(start_time)).first()
if score:
logger.debug('Received score for backup {}'.format(hashid))
task.set_status(GradingStatus.DONE)
elif task.expired(WAITING_TIMEOUT):
logger.warning(
'Did not receive score for backup {} in {} seconds, retrying'
.format(hashid, WAITING_TIMEOUT))
retry_task(task)
# report summary
statuses = collections.Counter(task.status for task in tasks)
message = '{} graded, {} failed'.format(statuses[GradingStatus.DONE],
statuses[GradingStatus.FAILED])
logger.info(message)
return message
def grade_on_effort(assignment_id, full_credit, late_multiplier, required_questions, grading_url):
logger = jobs.get_job_logger()
current_user = jobs.get_current_job().user
assignment = Assignment.query.get(assignment_id)
submissions = assignment.course_submissions(include_empty=False)
# archive all previous effort scores for this assignment
scores = Score.query.filter(
Score.kind == 'effort',
Score.assignment_id == assignment_id).all()
for score in scores:
db.session.delete(score)
seen = set()
stats = Counter()
manual, late, not_perfect = [], [], []
for i, subm in enumerate(submissions, 1):
user_id = int(subm['user']['id'])
if user_id in seen:
continue
latest_backup = Backup.query.get(subm['backup']['id'])
submission_time = get_submission_time(latest_backup, assignment)
backup, submission_time = find_best_scoring(latest_backup,
submission_time, assignment, required_questions, full_credit)
try:
score, messages = effort_score(backup, full_credit, required_questions)
except AssertionError:
manual.append(backup)
continue
else:
score, messages = handle_late(backup, assignment,
late, submission_time, score, messages, late_multiplier)
if score < full_credit and backup.hashid not in late:
not_perfect.append(backup)
messages.append('\nFinal Score: {}'.format(score))
messages.append('Your final score will be the max of either this score or the `Total` score (if exists)')
new_score = Score(score=score, kind='effort',
message='\n'.join(messages), user_id=backup.submitter_id,
assignment=assignment, backup=backup, grader=current_user)
db.session.add(new_score)
if i % 100 == 0:
logger.info('Scored {}/{}'.format(i, len(submissions)))
if subm['group']:
member_ids = {int(id) for id in subm['group']['group_member'].split(',')}
seen |= member_ids
stats[score] += len(member_ids)
else:
seen.add(user_id)
stats[score] += 1
# Commit all scores at once
db.session.commit()
logger.info('Scored {}/{}'.format(i, len(submissions)))
logger.info('done!')
if len(late) > 0:
logger.info('\n{} Late:'.format(len(late)))
for backup_id in late:
logger.info(' {}'.format(grading_url + backup_id))
logger.info('\nScore Distribution:')
sorted_scores = sorted(stats.items(), key=lambda p: -p[0])
for score, count in sorted_scores:
logger.info(' {} - {}'.format(str(score).rjust(3), count))
needs_autograding = len(manual) + len(not_perfect)
if needs_autograding > 0:
logger.info('\nAutograding {} manual and/or not perfect backups'.format(needs_autograding))
backup_ids = [backup.id for backup in manual + not_perfect]
try:
autograde_backups(assignment, current_user.id, backup_ids, logger)
except ValueError:
logger.info('Could not autograde backups - Please add an autograding key.')
db.session.commit()
return '/admin/course/{cid}/assignments/{aid}/scores'.format(
cid=jobs.get_current_job().course_id, aid=assignment_id)
def assign_scores(assign_id,
score,
kind,
message,
deadline,
include_backups=True,
grade_backups=False):
logger = jobs.get_job_logger()
current_user = jobs.get_current_job().user
assignment = Assignment.query.get(assign_id)
students = [e.user_id for e in assignment.course.get_students()]
submission_time = server_time_obj(deadline, assignment.course)
# Find all submissions (or backups) before the deadline
backups = Backup.query.filter(
Backup.assignment_id == assign_id,
or_(Backup.created <= deadline,
Backup.custom_submission_time <= deadline)).group_by(
Backup.submitter_id).order_by(Backup.created.desc())
if not include_backups:
backups = backups.filter(Backup.submit == True)
all_backups = backups.all()
if not all_backups:
logger.info("No submissions were found with a deadline of {}.".format(
deadline))
return "No Scores Created"
score_counter, seen = 0, set()
unique_backups = []
for back in all_backups:
if back.creator not in seen:
unique_backups.append(back)
seen |= back.owners()
total_count = len(unique_backups)
logger.info(
"Found {} unique and eligible submissions...".format(total_count))
if grade_backups:
logger.info('\nAutograding {} backups'.format(total_count))
backup_ids = [back.id for back in unique_backups]
try:
autograde_backups(assignment, current_user.id, backup_ids, logger)
except ValueError:
logger.info(
'Could not autograde backups - Please add an autograding key.')
else:
for back in unique_backups:
new_score = Score(score=score,
kind=kind,
message=message,
user_id=back.submitter_id,
assignment=assignment,
backup=back,
grader=current_user)
db.session.add(new_score)
new_score.archive_duplicates()
score_counter += 1
if score_counter % 100 == 0:
logger.info("Scored {} of {}".format(score_counter,
total_count))
# only commit if all scores were successfully added
db.session.commit()
logger.info("Left {} '{}' scores of {}".format(score_counter, kind.title(),
score))
return '/admin/course/{cid}/assignments/{aid}/scores'.format(
cid=jobs.get_current_job().course_id, aid=assignment.id)
def assign_scores(assign_id, score, kind, message, deadline,
include_backups=True, grade_backups=False):
logger = jobs.get_job_logger()
current_user = jobs.get_current_job().user
assignment = Assignment.query.get(assign_id)
students = [e.user_id for e in assignment.course.get_students()]
submission_time = server_time_obj(deadline, assignment.course)
# Find all submissions (or backups) before the deadline
backups = Backup.query.filter(
Backup.assignment_id == assign_id,
or_(Backup.created <= deadline, Backup.custom_submission_time <= deadline)
).group_by(Backup.submitter_id).order_by(Backup.created.desc())
if not include_backups:
backups = backups.filter(Backup.submit == True)
all_backups = backups.all()
if not all_backups:
logger.info("No submissions were found with a deadline of {}."
.format(deadline))
return "No Scores Created"
score_counter, seen = 0, set()
unique_backups = []
for back in all_backups:
if back.creator not in seen:
unique_backups.append(back)
seen |= back.owners()
total_count = len(unique_backups)
logger.info("Found {} unique and eligible submissions...".format(total_count))
if grade_backups:
logger.info('\nAutograding {} backups'.format(total_count))
backup_ids = [back.id for back in unique_backups]
try:
autograde_backups(assignment, current_user.id, backup_ids, logger)
except ValueError:
logger.info('Could not autograde backups - Please add an autograding key.')
else:
for back in unique_backups:
new_score = Score(score=score, kind=kind, message=message,
user_id=back.submitter_id,
assignment=assignment, backup=back,
grader=current_user)
db.session.add(new_score)
new_score.archive_duplicates()
score_counter += 1
if score_counter % 100 == 0:
logger.info("Scored {} of {}".format(score_counter, total_count))
# only commit if all scores were successfully added
db.session.commit()
logger.info("Left {} '{}' scores of {}".format(score_counter, kind.title(), score))
return '/admin/course/{cid}/assignments/{aid}/scores'.format(
cid=jobs.get_current_job().course_id, aid=assignment.id)
