def test_delete(self): """ Students should not be able to delete files. A deletion should work. """ self.assertFalse(ExternalFile.can(self.file1, self.user1, 'delete')) self.assertTrue(ExternalFile.can(self.file1, self.staff1, 'delete')) self.file1.delete() self.assertTrue(self.file1.deleted)
def test_delete(self): """ Students should not be able to delete files. A deletion should work. """ self.assertFalse(ExternalFile.can(self.file1, self.user1, 'delete')) self.assertTrue(ExternalFile.can(self.file1, self.staff1, 'delete')) self.file1.delete() self.assertTrue(self.file1.deleted)
def file_download(file_id, user): ext_file = ExternalFile.query.filter_by(id=file_id, deleted=False).first() if not ext_file or not ExternalFile.can(ext_file, user, 'download'): logger.info("Access file without permission by {0}".format(user.email)) abort(404) try: storage_obj = ext_file.object() except libcloud.common.types.InvalidCredsError: logger.warning("Could not get file {0} - {1}".format(file_id, ext_file.filename), exc_info=True) storage_obj = None if storage_obj is None: abort(404, "File does not exist") basename = os.path.basename(ext_file.filename) # Do not use .download_url for local storage. if storage.provider == libcloud.storage.types.Provider.LOCAL: response = Response(storage.get_object_stream(storage_obj), mimetype=ext_file.mimetype) response.headers["Content-Security-Policy"] = "default-src 'none';" response.headers["X-Content-Type-Options"] = "nosniff" response.headers["Content-Disposition"] = ("attachment; filename={0!s}" .format(basename)) return response else: postpend = '&' if request.args.get('raw'): postpend += urlencode({'response-content-type': ext_file.mimetype}) elif request.args.get('download'): postpend += urlencode({'response-content-disposition': 'attachment', 'filename': basename}) url = storage.get_blob_url(storage_obj.name) return redirect(url + postpend)
def test_duplicate_overwrite(self): with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: duplicate = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, course_id=self.course.id) duplicate_obj = duplicate.object() self.assertEquals(self.blob1.driver.key, duplicate_obj.driver.key) self.assertEquals(self.file1.filename, duplicate.filename) self.assertEquals(self.blob1.name, duplicate_obj.name) duplicate_obj.delete()
def test_duplicate_overwrite(self): with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: duplicate = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, course_id=self.course.id) duplicate_obj = duplicate.object() self.assertEqual(self.blob1.driver.key, duplicate_obj.driver.key) self.assertEqual(self.file1.filename, duplicate.filename) self.assertEqual(self.blob1.name, duplicate_obj.name) duplicate_obj.delete()
def setUp(self): super(TestFile, self).setUp() self.setup_course() with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: self.file1 = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, course_id=self.course.id) self.blob1 = self.file1.object() with open(CWD + "/../server/static/img/logo.svg", 'rb') as f: self.file2 = ExternalFile.upload(f, name='ok.svg', user_id=self.user1.id, course_id=self.course.id, assignment_id=self.assignment.id) self.blob2 = self.file1.object() db.session.add_all([self.file1, self.file2]) db.session.commit()
def setUp(self): super(TestFile, self).setUp() self.setup_course() with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: self.file1 = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, staff_file=True, course_id=self.course.id) self.blob1 = self.file1.object() with open(CWD + "/../server/static/img/logo.svg", 'rb') as f: self.file2 = ExternalFile.upload(f, name='ok.svg', user_id=self.user1.id, course_id=self.course.id, staff_file=False, assignment_id=self.assignment.id) self.blob2 = self.file2.object() db.session.add_all([self.file1, self.file2]) db.session.commit()
def test_malicious_directory_traversal(self): with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: prefix = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, prefix='test/../../', course_id=self.course.id) prefix_obj = prefix.object() self.assertEquals(self.blob1.driver.key, prefix_obj.driver.key) self.assertEquals(self.blob1.container.name, prefix.container) self.assertEquals(self.blob1.name, self.file1.object_name) self.assertEquals(prefix_obj.name, 'test_.._.._fizz.txt') self.assertEquals(prefix.filename, 'fizz.txt') prefix_obj.delete()
def test_malicious_directory_traversal(self): with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: prefix = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, prefix='test/../../', course_id=self.course.id) prefix_obj = prefix.object() self.assertEqual(self.blob1.driver.key, prefix_obj.driver.key) self.assertEqual(self.blob1.container.name, prefix.container) self.assertEqual(self.blob1.name, self.file1.object_name) self.assertEqual(prefix_obj.name, self.test_malicious_directory_traversal_expected_obj_name) self.assertEqual(prefix.filename, 'fizz.txt') prefix_obj.delete()
def test_prefix(self): with open(CWD + "/files/fizzbuzz_after.py", 'rb') as f: prefix = ExternalFile.upload(f, name='fizz.txt', user_id=self.staff1.id, prefix='test/', course_id=self.course.id) prefix_obj = prefix.object() self.assertEqual(self.blob1.driver.key, prefix_obj.driver.key) self.assertEqual(self.blob1.container.name, prefix.container) self.assertEqual(self.blob1.name, self.file1.object_name) self.assertEqual(prefix_obj.name, self.test_prefix_expected_obj_name) self.assertEqual(prefix.filename, 'fizz.txt') prefix_obj.delete()
def test_permission(self): # Students can not access files of staff self.assertTrue(ExternalFile.can(self.file1, self.staff1, 'download')) self.assertFalse(ExternalFile.can(self.file1, self.user1, 'download')) self.assertFalse(ExternalFile.can(self.file1, self.lab_assistant1, 'download')) # Staff and student can access student files self.assertTrue(ExternalFile.can(self.file2, self.user1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.staff1, 'download')) self.assertFalse(ExternalFile.can(self.file2, self.user2, 'download'))
def export_grades(): logger = jobs.get_job_logger() current_user = jobs.get_current_job().user course = Course.query.get(jobs.get_current_job().course_id) assignments = course.assignments students = (Enrollment.query.options(db.joinedload('user')).filter( Enrollment.role == STUDENT_ROLE, Enrollment.course == course).all()) headers, assignments = get_headers(assignments) logger.info("Using these headers:") for header in headers: logger.info('\t' + header) logger.info('') total_students = len(students) users = [student.user for student in students] user_ids = [user.id for user in users] all_scores = collect_all_scores(assignments, user_ids) with io.StringIO() as f: writer = csv.writer(f) writer.writerow(headers) # write headers for i, student in enumerate(students, start=1): row = export_student_grades(student, assignments, all_scores) writer.writerow(row) if i % 50 == 0: logger.info('Exported {}/{}'.format(i, total_students)) f.seek(0) created_time = local_time(dt.datetime.now(), course, fmt='%b-%-d %Y at %I-%M%p') csv_filename = '{course_name} Grades ({date}).csv'.format( course_name=course.display_name, date=created_time) # convert to bytes for csv upload csv_bytes = io.BytesIO(bytearray(f.read(), 'utf-8')) upload = ExternalFile.upload(csv_bytes, user_id=current_user.id, name=csv_filename, course_id=course.id, prefix='jobs/exports/{}/'.format( course.offering)) logger.info('\nDone!\n') logger.info("Saved as: {0}".format(upload.object_name)) return "/files/{0}".format(encode_id(upload.id))
def test_permission(self): # Students can not access files of staff self.assertTrue(ExternalFile.can(self.file1, self.staff1, 'download')) self.assertFalse(ExternalFile.can(self.file1, self.user1, 'download')) self.assertFalse(ExternalFile.can(self.file1, self.lab_assistant1, 'download')) # Staff and student can access student files self.assertTrue(ExternalFile.can(self.file2, self.user1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.staff1, 'download')) self.assertFalse(ExternalFile.can(self.file2, self.user2, 'download'))
def test_group_permission(self): Group.invite(self.user1, self.user2, self.assignment) group = Group.lookup(self.user1, self.assignment) # Only the original creator and staff can accept the files self.assertTrue(ExternalFile.can(self.file2, self.user1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.staff1, 'download')) self.assertFalse(ExternalFile.can(self.file2, self.user2, 'download')) group.accept(self.user2) # Now all group members can access the files self.assertTrue(ExternalFile.can(self.file2, self.user1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.staff1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.user2, 'download')) self.assertFalse(ExternalFile.can(self.file2, self.user3, 'download'))
def test_group_permission(self): Group.invite(self.user1, self.user2, self.assignment) group = Group.lookup(self.user1, self.assignment) # Only the original creator and staff can accept the files self.assertTrue(ExternalFile.can(self.file2, self.user1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.staff1, 'download')) self.assertFalse(ExternalFile.can(self.file2, self.user2, 'download')) group.accept(self.user2) # Now all group members can access the files self.assertTrue(ExternalFile.can(self.file2, self.user1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.staff1, 'download')) self.assertTrue(ExternalFile.can(self.file2, self.user2, 'download')) self.assertFalse(ExternalFile.can(self.file2, self.user3, 'download'))
def test_job(duration=0, should_fail=False, make_file=False): logger = jobs.get_job_logger() logger.info('Starting...') time.sleep(duration) if should_fail: 1/0 if make_file: upload = ExternalFile.upload(data(duration+1), user_id=1, course_id=1, name='temp.okfile', prefix='jobs/example/') logger.info("Saved as: {}".format(upload.object_name)) logger.info('File ID: {0}'.format(encode_id(upload.id))) msg = ("Waited for <a href='/files/{0}'> {1} seconds </a>" .format(encode_id(upload.id), duration)) else: msg = "Waited for <b>{}</b> seconds!".format(duration) logger.info('Finished!') return msg
def export_assignment(assignment_id, anonymized): """ Generate a zip file of submissions from enrolled students. Final Submission: One submission per student/group Zip Strucutre: cal-cs61a../[email protected]@b.com/abc12d/hog.py Anonymized: Submission without identifying info Zip Strucutre: cal-cs61a../{hash}/hog.py """ logger = jobs.get_job_logger() assignment = Assignment.query.get(assignment_id) requesting_user = jobs.get_current_job().user if not assignment: logger.warning("No assignment found") raise Exception("No Assignment") if not Assignment.can(assignment, requesting_user, "download"): raise Exception("{} does not have enough permission" .format(requesting_user.email)) if anonymized: logger.info("Starting anonymized submission export") else: logger.info("Starting final submission export") course = assignment.course with io.BytesIO() as bio: # Get a handle to the in-memory zip in append mode with zipfile.ZipFile(bio, "w", zipfile.ZIP_DEFLATED, False) as zf: zf.external_attr = 0o655 << 16 export_loop(bio, zf, logger, assignment, anonymized) created_time = local_time(dt.datetime.now(), course, fmt='%m-%d-%I-%M-%p') zip_name = '{}_{}.zip'.format(assignment.name.replace('/', '-'), created_time) bio.seek(0) # Close zf handle to finish writing zipfile logger.info("Uploading...") upload = ExternalFile.upload(bio, user_id=requesting_user.id, name=zip_name, course_id=course.id, prefix='jobs/exports/{}/'.format(course.offering)) logger.info("Saved as: {0}".format(upload.object_name)) msg = "/files/{0}".format(encode_id(upload.id)) return msg
def test_job(duration=0, should_fail=False, make_file=False): logger = jobs.get_job_logger() logger.info('Starting...') time.sleep(duration) if should_fail: 1 / 0 if make_file: upload = ExternalFile.upload(data(duration + 1), user_id=1, course_id=1, name='temp.okfile', prefix='jobs/example/') logger.info("Saved as: {}".format(upload.object_name)) logger.info('File ID: {0}'.format(encode_id(upload.id))) msg = ("Waited for <a href='/files/{0}'> {1} seconds </a>".format( encode_id(upload.id), duration)) else: msg = "Waited for <b>{}</b> seconds!".format(duration) logger.info('Finished!') return msg
def export_grades(): logger = jobs.get_job_logger() current_user = jobs.get_current_job().user course = Course.query.get(jobs.get_current_job().course_id) assignments = course.assignments students = (Enrollment.query .options(db.joinedload('user')) .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) .all()) headers, assignments = get_headers(assignments) logger.info("Using these headers:") for header in headers: logger.info('\t' + header) logger.info('') total_students = len(students) with io.StringIO() as f: writer = csv.writer(f) writer.writerow(headers) # write headers for i, student in enumerate(students, start=1): row = export_student_grades(student, assignments) writer.writerow(row) if i % 50 == 0: logger.info('Exported {}/{}'.format(i, total_students)) f.seek(0) created_time = local_time(dt.datetime.now(), course, fmt='%b-%-d %Y at %I-%M%p') csv_filename = '{course_name} Grades ({date}).csv'.format( course_name=course.display_name, date=created_time) # convert to bytes for csv upload csv_bytes = io.BytesIO(bytearray(f.read(), 'utf-8')) upload = ExternalFile.upload(csv_bytes, user_id=current_user.id, name=csv_filename, course_id=course.id, prefix='jobs/exports/{}/'.format(course.offering)) logger.info('\nDone!\n') logger.info("Saved as: {0}".format(upload.object_name)) return "/files/{0}".format(encode_id(upload.id))
def file_download(file_id, user): ext_file = ExternalFile.query.filter_by(id=file_id, deleted=False).first() if not ext_file or not ExternalFile.can(ext_file, user, 'download'): logger.info("Access file without permission by {0}".format(user.email)) abort(404) try: storage_obj = ext_file.object() except libcloud.common.types.InvalidCredsError: logger.warning("Could not get file {0} - {1}".format( file_id, ext_file.filename), exc_info=True) storage_obj = None if storage_obj is None: abort(404, "File does not exist") basename = os.path.basename(ext_file.filename) # Do not use .download_url for local storage. if storage.provider == libcloud.storage.types.Provider.LOCAL: response = Response(storage.get_object_stream(storage_obj), mimetype=ext_file.mimetype) response.headers["Content-Security-Policy"] = "default-src 'none';" response.headers["X-Content-Type-Options"] = "nosniff" response.headers["Content-Disposition"] = ( "attachment; filename={0!s}".format(basename)) return response else: postpend = '&' if request.args.get('raw'): postpend += urlencode({'response-content-type': ext_file.mimetype}) elif request.args.get('download'): postpend += urlencode({ 'response-content-disposition': 'attachment', 'filename': basename }) url = storage.get_blob_url(storage_obj.name) return redirect(url + postpend)
def submit_assignment(name): # TODO: Unify student & staff upload. assign = get_assignment(name) group = Group.lookup(current_user, assign) user_ids = assign.active_user_ids(current_user.id) fs = assign.final_submission(user_ids) if not assign.uploads_enabled: flash("This assignment cannot be submitted online", 'warning') return redirect(url_for('.assignment', name=assign.name)) extension = None # No need for an extension if not assign.active: extension = Extension.get_extension(current_user, assign) if not extension: flash("It's too late to submit this assignment", 'warning') return redirect(url_for('.assignment', name=assign.name)) if request.method == "POST": backup = Backup.create( submitter=current_user, assignment=assign, submit=True, ) assignment = backup.assignment if extension: backup.custom_submission_time = extension.custom_submission_time templates = assignment.files or [] files = {} def extract_file_index(file_ind): """ Get the index of of file objects. Used because request.files.getlist() does not handle uniquely indexed lists. >>> extract_file_index('file[12']) 12 """ brace_loc = file_ind.find('[') index_str = file_ind[brace_loc+1:-1] return int(index_str) # A list of one element lists sorted_uploads = sorted(list(request.files.items()), key=lambda x: extract_file_index(x[0])) uploads = [v[1] for v in sorted_uploads] full_path_names = list(request.form.listvalues())[0] template_files = assign.files or [] file_names = [os.path.split(f)[1] for f in full_path_names] missing = [t for t in template_files if t not in file_names] if missing: return jsonify({ 'error': ('Missing files: {}. The following files are required: {}' .format(', '.join(missing), ', '.join(template_files))) }), 400 backup_folder_postfix = time.time() for full_path, upload in zip(full_path_names, uploads): data = upload.read() if len(data) > MAX_UPLOAD_FILE_SIZE: # file is too large (over 25 MB) return jsonify({ 'error': ('{} is larger than the maximum file size of {} MB' .format(full_path, MAX_UPLOAD_FILE_SIZE/1024/1024)) }), 400 try: files[full_path] = str(data, 'utf-8') except UnicodeDecodeError: upload.stream.seek(0) # We've already read data, so reset before uploading dest_folder = "uploads/{}/{}/{}/".format(assign.name, current_user.id, backup_folder_postfix) bin_file = ExternalFile.upload(upload.stream, current_user.id, full_path, staff_file=False, prefix=dest_folder, course_id=assign.course.id, backup=backup, assignment_id=assign.id) db.session.add(bin_file) message = Message(kind='file_contents', contents=files) backup.messages.append(message) db.session.add(backup) db.session.commit() # Send to continuous autograder if assign.autograding_key and assign.continuous_autograding: try: submit_continuous(backup) except ValueError as e: flash('Did not send to autograder: {}'.format(e), 'warning') return jsonify({ 'backup': backup.hashid, 'url': url_for('.code', name=assign.name, submit=backup.submit, bid=backup.id) }) return render_template('student/assignment/submit.html', assignment=assign, group=group, course=assign.course)