def submit_autoencoder(submission, score, email, token):
grader_ = grader.Grader("9TShnp1JEeeGGAoCUnhvuA")
encoder, decoder = submission
grader_.set_answer("FtBSK", encoder.output_shape[1])
grader_.set_answer("83Glu", decoder.output_shape[1:])
grader_.set_answer("fnM1K", score)
grader_.submit(email, token)
def test_Wes(self):
grader = g.Grader()
jsonData = grader.grade('test/images/mystery_tests/Wes.jpg',
self.debug_mode, self.verbose_mode, 1.0, 'act',
1)
data = json.loads(jsonData)
expected_answers = [
('D J D H D J B J B G B J B ' + 'H D H B F B H A F B J B F ' +
'D H A G D G B G A F A F C ' + 'J C G D J C F C H A F D J ' +
'A F B G C F D G C H D H D ' + 'H A G B J A G B F D').split(' '),
('D H E F E H C H A K ' + 'A K B H A G D F A F ' +
'B H A H B G B F D G ' + 'D H B J D J A J B F ' +
'C J E G C F E H B J ' + 'E G B F D J C J B K').split(' '),
('A J C H B F D ' + 'H B H A J C J ' + 'B H B F D F A ' +
'G D J B G A J ' + 'A H A G A F C ' + 'F A G B H').split(' '),
('C G D G C F A ' + 'F D F B J D J ' + 'C J B H B J C ' +
'F B J B G A G ' + 'B F A G D F A ' + 'F B F A F').split(' ')
]
self.assertIn(
'boxes', data,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
self.assertGreater(
len(data['boxes']), 0,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
for i, graded_results in enumerate(data['boxes']):
print(f'ACT test {i+1}')
self.assertEqual(len(graded_results['results']['bubbled']),
len(expected_answers[i]))
self.assertEqual(
list(graded_results['results']['bubbled'].values()),
expected_answers[i])
def test_act_page1(self):
grader = g.Grader()
jsonData = grader.grade('test/images/act_test1.JPG', self.debug_mode,
self.verbose_mode, 1.0, 'act', 1)
data = json.loads(jsonData)
expected_answers = [
('A J D H B J B J C J B G A ' + 'H D H B H C H A F B H C F ' +
'D G A F B H B H A F A F C ' + 'J C G D J C J D H B F D J ' +
'A F D G C F D G C H A F C ' + 'H D G D J A F B H D').split(' '),
('D H E F E H E H A K ' + 'C K B H B H D F D F ' +
'B H A H B G A H C G ' + 'E K B H D J A F B F ' +
'E K D G D J B G A F ' + 'E H B K E K A K E J').split(' '),
('A J C H B J A ' + 'H B J A H C J ' + 'B H B F D F A ' +
'G D J C G C J ' + 'C G A G A J D ' + 'H B J A H').split(' '),
('C G D G C F A ' + 'F D J C F B F ' + 'C F B H B J D ' +
'F B J B H A J ' + 'C F B G D J A ' + 'H B F D J').split(' ')
]
self.assertIn(
'boxes', data,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
self.assertGreater(
len(data['boxes']), 0,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
for i, graded_results in enumerate(data['boxes']):
print(f'ACT test {i+1}')
self.assertEqual(len(graded_results['results']['bubbled']),
len(expected_answers[i]))
self.assertEqual(
list(graded_results['results']['bubbled'].values()),
expected_answers[i])
def test_Dep(self):
grader = g.Grader()
jsonData = grader.grade('test/images/mystery_tests/Dep.jpg',
self.debug_mode, self.verbose_mode, 1.0, 'act',
1)
data = json.loads(jsonData)
expected_answers = [
('A H D J C J B J D J B G F ' + 'H D G B H C H A F B H C F ' +
'D G A F B G B H A F D F C ' + 'J C G D J C J B H B F B J ' +
'A F B G C F D G C F C H C ' + 'H C H C H C H C H C').split(' '),
('D H D F E H E H A K ' + 'C K B H B H D F D F ' +
'B H A J B G E H C G ' + 'D K E H D G A F B F ' +
'E K D G D J F G D F ' + 'E K D G E G B H E G').split(' '),
('A G B H B F A ' + 'H B H A J D H ' + 'C G B J B F - ' +
'- B H C H C H ' + 'C H C H C H C ' + 'H B H C H').split(' '),
('C G D G C F A ' + 'F D J C F B J ' + 'C F B J B G D ' +
'J C G F H A G ' + 'C F B G D H B ' + 'H A H A H').split(' ')
]
self.assertIn(
'boxes', data,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
self.assertGreater(
len(data['boxes']), 0,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
for i, graded_results in enumerate(data['boxes']):
print(f'ACT test {i+1}')
self.assertEqual(len(graded_results['results']['bubbled']),
len(expected_answers[i]))
self.assertEqual(
list(graded_results['results']['bubbled'].values()),
expected_answers[i])
def test_act_page1b(self):
grader = g.Grader()
jsonData = grader.grade('test/images/act_test1b.png', self.debug_mode,
self.verbose_mode, 1.0, 'act', 1)
data = json.loads(jsonData)
expected_answers = [
('A J D H B J B J C J B G A ' + 'H D H D H C G A F B H D F ' +
'D J A J D G B H A F A F C ' + 'J C G C F C J C H B G B J ' +
'D F D F C F B H C H A F C ' + 'H C H C H C H C H C').split(' '),
('D H E F E H E H A K ' + 'C K C G B H D F D F ' +
'B F A H B G E H C F ' + 'D K B H E H A F B F ' +
'D K D G E J B G D F ' + 'E H D K E H D J E G').split(' '),
('D J C H B J A ' + 'H B J A J C F ' + 'B H B F D F B ' +
'H D H C F C J ' + 'A H A G A J A ' + 'J B G B F').split(' '),
('C G D G C F C ' + 'F D J C H C F ' + 'C F B H A J D ' +
'F B G A G A J ' + 'D F B G D J A ' + 'H C G A J').split(' ')
]
self.assertIn(
'boxes', data,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
self.assertGreater(
len(data['boxes']), 0,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
for i, graded_results in enumerate(data['boxes']):
print(f'ACT test {i+1}')
self.assertEqual(len(graded_results['results']['bubbled']),
len(expected_answers[i]))
self.assertEqual(
list(graded_results['results']['bubbled'].values()),
expected_answers[i])
def test_act_simple(self):
answer_counts = [75, 60, 40, 40]
for testsuffix in ['d', 'f', 'g', 'h']:
print(f'act test 1{testsuffix}')
grader = g.Grader()
jsonData = grader.grade(f'test/images/act_test1{testsuffix}.jpg',
self.debug_mode, self.verbose_mode, 1.0,
'act', 1)
data = json.loads(jsonData)
self.assertIn(
'boxes', data,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
self.assertGreater(
len(data['boxes']), 0,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
for i, graded_results in enumerate(data['boxes']):
print(f'ACT test 1{testsuffix} box {i+1}')
self.assertEqual(len(graded_results['results']['bubbled']),
answer_counts[i],
len(graded_results['results']['bubbled']))
blank_answers_num = len([
a for a in graded_results['results']['bubbled'] if a == '-'
])
self.assertEqual(blank_answers_num, 0)
def test_act_page1a(self):
grader = g.Grader()
jsonData = grader.grade('test/images/act_test1a.jpg', self.debug_mode,
self.verbose_mode, 1.0, 'act', 1)
data = json.loads(jsonData)
expected_answers = [
('A J D H B J B F D J B G A ' + 'H D H D F A H A F B H A G ' +
'D F A F B G D H A F A F C ' + 'J B G C J D J C H B G D F ' +
'A F B G C F A G B G A F C ' + 'F D G B H D F - - -').split(' '),
('D H D F E H E H A K ' + 'C K B H D H A F D F ' +
'B H A H B G E H - H ' + 'D K B G C G A F B F ' +
'D J D G D G B F D G ' + 'E J B H D K C G B H').split(' '),
('A J B H B H A ' + 'H B J B J C J ' + 'B F B H D F A ' +
'G C J B J C J ' + 'C G A H B H A ' + '- - - - -').split(' '),
('C G D G B F A ' + 'F D H C J B F ' + 'C K A H B J D ' +
'F B J B H A G ' + 'C F B G D G B ' + 'G C H B H').split(' ')
]
self.assertIn(
'boxes', data,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
self.assertGreater(
len(data['boxes']), 0,
f"We couldn't find any boxes because we enountered this error: {data['error']}"
)
for i, graded_results in enumerate(data['boxes']):
print(f'ACT test {i+1}')
self.assertEqual(len(graded_results['results']['bubbled']),
len(expected_answers[i]))
self.assertEqual(
list(graded_results['results']['bubbled'].values()),
expected_answers[i])
def submit_char_rnn(submission, score, email, token):
grader_ = grader.Grader("cULEpp2NEeemQBKZKgu93A")
history, samples = submission
assert len(samples) == 25
grader_.set_answer("pttMO",
int(np.mean(history[:10]) > np.mean(history[-10:])))
grader_.set_answer("uly0D", len(set(samples)))
grader_.submit(email, token)
def dreadnought(test, page, box, imgpath):
click.echo(f'Grading {imgpath} ({test} page {page} box {box})...')
"""
"""
grader = g.Grader()
jsonData = grader.grade(imgpath, False, False, 1.0, test.lower(), box,
page)
data = json.loads(jsonData)
print(data['answer']['bubbled'])
def test_Grader():
# CONSIDER: This testing is inadequate. Should we take the time
# to make unit-tests for the classes/methods in this project?
# TODO: At the least, test (informally) better than the following.
csse120 = course.CSSE120
what_to_grade = grader.ThingToGrade(csse120, 0) # Session 16
grader_for_testing = grader.Grader(what_to_grade)
grader_for_testing.grade()
def test_page1(self):
grader = g.Grader()
jsonData = grader.grade('test/images/sat_test1.jpg', self.debug_mode, self.verbose_mode, 1.0, 'sat', 1)
data = json.loads(jsonData)
self.assertIsNotNone(data['boxes'])
self.assertEqual(len(list(data['boxes'][0]['results']['bubbled'].values())),52)
self.assertEqual(list(data['boxes'][0]['results']['bubbled'].values()), 'B C A A C A A B B D D D C'.split(' ') +
'B B C B B A A D A A B C C'.split(' ') +
'B B D D B C C D C A D C A'.split(' ') +
'D A C C D D C B B A B D D'.split(' ') )
def test_page4(self):
grader = g.Grader()
jsonData = grader.grade('test/images/sat_test4.jpg', self.debug_mode, self.verbose_mode, 1.0, 'sat', 4)
data = json.loads(jsonData)
self.assertEqual(len(list(data['boxes'][0]['results']['bubbled'].values())),30 )
self.assertEqual(list(data['boxes'][0]['results']['bubbled'].values()), 'C B C C B A'.split(' ') +
'D C B D B D'.split(' ') +
'D A A B B B'.split(' ') +
'C B C - C -'.split(' ') +
'D C C D A A'.split(' ') )
def test_quiz_1_19(self):
grader = g.Grader()
jsonData = grader.grade('images/6q/set_1/calQuiz_Page_19.png', False,
False, 1.0)
data = json.loads(jsonData)
self.assertEqual(data['answer']['bubbled'],
['A', 'B', '-', '-', '-', '-'])
self.assertEqual(data['version']['bubbled'], ['-'])
self.assertEqual(data['id']['bubbled'],
['0', '1', '6', '1', '8', '7', '1', '4', '1'])
def test_quiz_3_39(self):
grader = g.Grader()
jsonData = grader.grade('images/6q/set_3/image-39.png', False, False,
1.0)
data = json.loads(jsonData)
self.assertEqual(data['answer']['bubbled'],
['-', '-', '-', '-', '-', '-'])
self.assertEqual(data['version']['bubbled'], ['-'])
self.assertEqual(data['id']['bubbled'],
['0', '1', '5', '9', '7', '3', '2', '6', '5'])
def test_quiz_1_08(self):
grader = g.Grader()
jsonData = grader.grade('images/6q/set_1/calQuiz_Page_08.png', False,
False, 1.0)
data = json.loads(jsonData)
self.assertEqual(data['answer']['bubbled'],
['C', 'C', '-', '-', '-', '-'])
self.assertEqual(data['version']['bubbled'], ['-'])
self.assertEqual(data['id']['bubbled'],
['0', '1', '5', '9', '5', '1', '3', '3', '4'])
def test_quiz_3_23(self):
grader = g.Grader()
jsonData = grader.grade('images/6q/set_3/image-23.png', False, False,
1.0)
data = json.loads(jsonData)
self.assertEqual(data['answer']['bubbled'],
['-', '-', '-', '-', '-', '-'])
self.assertEqual(data['version']['bubbled'], ['-'])
self.assertEqual(data['id']['bubbled'],
['-', '-', '-', '-', '-', '-', '-', '-', '-'])
def test_quiz_3_08(self):
grader = g.Grader()
jsonData = grader.grade('images/6q/set_3/image-08.png', False, False,
1.0)
data = json.loads(jsonData)
self.assertEqual(data['answer']['bubbled'],
['D', 'A', '-', '-', '-', '-'])
self.assertEqual(data['version']['bubbled'], ['-'])
self.assertEqual(data['id']['bubbled'],
['0', '1', '4', '5', '8', '9', '0', '5', '1'])
def test_quiz_2_28(self):
grader = g.Grader()
jsonData = grader.grade('images/6q/set_2/quiz2_Page_28.png', False,
False, 1.0)
data = json.loads(jsonData)
self.assertEqual(data['answer']['bubbled'],
['B', 'A', 'B', '-', '-', '-'])
self.assertEqual(data['version']['bubbled'], ['-'])
self.assertEqual(data['id']['bubbled'],
['0', '1', '6', '3', '3', '6', '5', '2', '4'])
def test_page2(self):
grader = g.Grader()
jsonData = grader.grade('test/images/sat_test2.jpg', self.debug_mode, self.verbose_mode, 1.0, 'sat', 2)
data = json.loads(jsonData)
self.assertEqual(len(list(data['boxes'][0]['results']['bubbled'].values())),44)
self.assertEqual(list(data['boxes'][0]['results']['bubbled'].values()),'A B C C A B A D C'.split(' ') +
'C B A C D B C C B'.split(' ') +
'D C D A A D B A D'.split(' ') +
'B A A D B C C B C'.split(' ') +
'B C C B D A D D'.split(' ') )
def test_Hut5(self):
grader = g.Grader()
jsonData = grader.grade('test/images/imageHut35.jpeg', self.debug_mode, self.verbose_mode, 1.0, 'sat', 5)
data = json.loads(jsonData)
expected_answers = [ [102.0, 2.0, 30.0, 25.4, 2.0],
[7.0, 576.0, 0.8]
]
for i, graded_results in enumerate(data['boxes']):
self.assertEqual(len(graded_results['results']['bubbled']),len(expected_answers[i]))
self.assertEqual(list(graded_results['results']['bubbled'].values()), expected_answers[i])
def test_Mcc3(self):
grader = g.Grader()
jsonData = grader.grade('test/images/imageMcc33.jpeg', self.debug_mode, self.verbose_mode, 1.0, 'sat', 3)
data = json.loads(jsonData)
expected_answers = [ ( 'D A A'.split(' ') +
'B B D'.split(' ') +
'B C A'.split(' ') +
'C D C'.split(' ') +
'B A B'.split(' ') ) ,
['-', '-', 3.0, 5.0, '-']
]
for i, graded_results in enumerate(data['boxes']):
self.assertEqual(len(graded_results['results']['bubbled']),len(expected_answers[i]))
self.assertEqual(list(graded_results['results']['bubbled'].values()), expected_answers[i])
def __init__(self, master=None):
Frame.__init__(self, master)
self.pack()
# buttons
self.QUIT = Button(self)
self.openKeyButton = Button(self)
self.openAssignmentsButton = Button(self)
self.gradeButton = Button(self)
self.outputText = StringVar("")
self.output = Label(self, textvariable=self.outputText, relief=RAISED)
# button styling
self.setQuit()
self.setOpenKey()
self.setOpenAssignments()
self.setGrade()
self.setOutput()
self.grader = grader.Grader()
def submit_mse(compute_mse, email, token):
ASSIGNMENT_KEY = "SBaWP48eEeeGSBKyliRlgg"
PART_KEY = "u2t7D"
# First, do rigorous local testing to help the learner
for n in [1, 5, 10, 10**3]:
elems = [
np.arange(n),
np.arange(n, 0, -1),
np.zeros(n),
np.ones(n),
np.random.random(n),
np.random.randint(100, size=n)
]
for el in elems:
for el_2 in elems:
true_mse = np.array(mean_squared_error(el, el_2))
my_mse = compute_mse(el, el_2)
if not np.allclose(true_mse, my_mse):
print('mse(%s,%s)' % (el, el_2))
print("should be: %f, but your function returned %f" %
(true_mse, my_mse))
raise ValueError('Wrong result')
# Second, submit some reference values. There is nothing preventing the learner from
# manually submitting numbers computed not via tensorflow, so there is little point
# in comprehensive server-side testing
test_pairs = ((np.array([
0.85415937, 0.768366, 0.9763879, 0.11861405, 0.21219242
]), np.array([0.27163543, 0.14893905, 0.84616464, 0.86294942,
0.65509213])), (np.array([1, 2, 3]), np.array([3, 2, 2])),
(np.array([1]), np.array([1])))
answers = []
for pair in test_pairs:
answers.append(compute_mse(pair[0], pair[1]))
grade_submitter = grader.Grader(ASSIGNMENT_KEY)
grade_submitter.set_answer(PART_KEY, answers)
grade_submitter.submit(email, token)
def grade_test(examinfo, send_email_flag):
try:
usererrors = []
adminerrors = []
imgurls = examinfo['Image Urls']
test = examinfo['Test']
email = examinfo["Email"], examinfo["Student Email"]
name = f'{examinfo["First Name"]} {examinfo["Last Name"]}'
page_answers = OrderedDict()
print(f'trying to grade from: {email} {test}')
for i, imgurl in enumerate(imgurls):
page = i + 1
with tempfile.TemporaryDirectory() as tmpdirname:
imgpath = f'{tmpdirname}/tmpimg'
with open(imgpath, 'wb') as imgfile:
download_success = download_image(imgurl, imgfile)
if download_success:
filetype = magic.from_file(imgpath, mime=True)
if not (filetype == "image/png"
or filetype == "image/jpeg"):
usererrors.append('unsupported_image_format')
break
print(
f'Wrote image into temporary file succesfully. Grading page {page}'
)
grader = g.Grader()
jsonData = grader.grade(imgpath, False, False, 1.0,
test.lower(), page, imgurl)
data = json.loads(jsonData)
if data['status'] == 0:
for box in data['boxes']:
print(box['results']['bubbled'])
if not box['name'] in page_answers:
page_answers[box['name']] = OrderedDict()
page_answers[box['name']].update(
box['results']['bubbled'])
elif data['status'] == 2:
usererrors.append(data['error'])
break
else:
adminerrors.append(data['error'])
else:
adminerrors.append('Unable to download {imgurl}')
if len(adminerrors) == 0 and len(usererrors) == 0:
if not DB_SERVER_NAME == 'skipdb':
print(
'No admin errors or user errors, uploading to database meow.'
)
upload_to_database(examinfo, page_answers)
subject, body = format_email_message('succesful_submission', {
'test': test,
'name': name
})
send_email(email, subject, [body], send_email_flag)
else:
print(
f'We got some errors, not adding to database: adminerrors: {adminerrors}, usererrors: {usererrors}'
)
if len(usererrors) > 0:
try:
subject, body = format_email_message(
usererrors[0], {
'test': test,
'name': name
})
except:
#this means we got a non-specific user error
subject, body = format_email_message(
'unhandled_image_error', {
'test': test,
'name': name
})
send_email(email, subject, [body], send_email_flag)
elif len(adminerrors) > 0:
subject, body = format_email_message('unhandled_image_error', {
'test': test,
'name': name
})
send_email(email, subject, [body], send_email_flag)
send_email(adminemail, 'Grader System Error', adminerrors,
send_email_flag)
else:
send_email(
adminemail, 'Crazy Town Error',
[f'How did we get here? {[traceback.format_exc()]}'],
send_email_flag)
except:
send_email(adminemail, 'Crazy Town Error',
[f'How did we get here? {[traceback.format_exc()]}'],
send_email_flag)
def make_Grader():
csse120 = course.CSSE120
what_to_grade = grader.ThingToGrade(csse120, 2)
who_to_grade = 'usernames-for_testing.txt'
return grader.Grader(what_to_grade, who_to_grade)
