def test_ABD(self):
grader = grading.Grader()
taskResponse = Mock()
taskResponse.graded_response = True
self.models.TaskResponse.query.filter_by.return_value.first.return_value = taskResponse
ret = grader.grade_automatic_questions(1)
self.assertEqual(ret, None)
def submit_mountain_car(generate_session, email, token):
sessions = [generate_session() for _ in range(100)]
_, _, session_rewards = zip(*sessions)
session_rewards = np.array(session_rewards)
grader = grading.Grader("EyYJW9bUEeeXyQ5ZPWKHGg")
grader.set_answer("mXDUE", np.mean(session_rewards))
grader.submit(email, token)
def get(self, responseID):
grader = grading.Grader()
grader.grade_automatic_questions(responseID)
grader.grade_supplementary_material(responseID)
task_response = models.TaskResponse.query.filter_by(
id=responseID).first()
course = task_response.task.course
courses_where_ta = current_user.get_courses_where_ta()
if task_response.student_id != current_user.id and \
course not in current_user.get_courses_where_teacher_or_ta():
return "Permission Denied", 401
if task_response.student_id == current_user.id and \
task_response.task.status != "grades published":
return "Task Not Yet Released"
formatted_time = task_response.datetime.strftime("%a %b %d %H:%M:%S")
response = json.loads(task_response.graded_response)
supplementary = json.loads(task_response.graded_supplementary)
supplementary_order = json.loads(task_response.supplementary_order)
return render_template("grading/response.html",
response=response,
student=task_response.user,
task=task_response.task,
task_response=task_response,
time=formatted_time,
supplementary=supplementary,
permissions=current_user.permissions,
supplementaryOrder=supplementary_order)
def submit_kungfu(agent, env, evaluate, email, token):
with tqdm.trange(10, desc='Evaluating your agent') as t:
sessions = [evaluate(agent=agent, env=env, n_games=1) for _ in t]
session_rewards = np.array(sessions)
grader = grading.Grader("6sPnVCn6EeieSRL7rCBNJA")
grader.set_answer("HhNVX", int(np.mean(session_rewards)))
grader.submit(email, token)
def test_has_questions(self):
trquestion1 = {
'questionID': 1,
'correctOption': "A",
'correctOptionText': "text"
}
question1 = {
'questionID': 1,
'selectedOption': "A",
}
taskResponse = Mock()
taskResponse.graded_response = False
taskResponse.response = json.dumps({
'automatic_questions': [question1],
'manual_questions': []
})
taskResponse.correctness_grade = None
taskResponse.task = Mock()
taskResponse.task.questions = json.dumps([trquestion1])
self.models.TaskResponse.query.filter_by.return_value.first.return_value = taskResponse
grader = grading.Grader()
grader.calculate_correctness = Mock()
grader.calculate_correctness.return_value = "grade"
ret = grader.grade_automatic_questions(1)
self.assertEqual(taskResponse.correctness_grade, "grade")
exp = {
'questionID': 1,
'selectedOption': "A",
'correctOption': "A",
'correct': True,
'correctOptionText': "text"
}
self.assertEqual(ret, [exp])
def submit_qlearning2(rewards, email, token):
flag1 = np.mean(rewards[-10:])
grader = grading.Grader("XbjcGd7xEeeDzRKutDCmyA")
grader.set_answer("CkyJ4", flag1)
grader.submit(email, token)
def submit_kungfu(session_rewards, email, token):
grader = grading.Grader("6sPnVCn6EeieSRL7rCBNJA")
# NOTE: Bug, so multiplied with 100, see
# https://www.coursera.org/learn/practical-rl/programming/95h2n/grader-a3c/discussions/threads/Q1XYCe08EeigTgpIFoGjKg
# https://www.coursera.org/learn/practical-rl/programming/95h2n/grader-a3c/discussions/threads/F1JHgGRmEei8iRK3RpIYPA/replies/vILx4WchEeimAgoMPemTPA
grader.set_answer("HhNVX", 100 * int(np.mean(session_rewards)))
grader.submit(email, token)
def submit_cartpole(generate_session, email, token):
sessions = [generate_session() for _ in range(100)]
# session_rewards, _, _ = map(np.array, zip(*sessions))
session_rewards = np.array(sessions)
grader = grading.Grader("RDofv-QXEeeaGw6kpIOf3g")
grader.set_answer("NRNkl", int(np.mean(session_rewards)))
grader.submit(email, token)
def submit_taxi(generate_session, policy, email, token):
sessions = [generate_session(policy) for _ in range(100)]
_, _, session_rewards = zip(*sessions)
session_rewards = np.array(session_rewards)
grader = grading.Grader("s4pTlNbTEeeQvQ7N1-Sa3A")
grader.set_answer("GsMSL", np.mean(session_rewards))
grader.submit(email, token)
def submit_autoencoder(submission, score, email, token):
grader = grading.Grader("9TShnp1JEeeGGAoCUnhvuA")
encoder, decoder = submission
grader.set_answer("FtBSK", encoder.output_shape[1])
grader.set_answer("83Glu", decoder.output_shape[1:])
grader.set_answer("fnM1K", score)
grader.submit(email, token)
def submit_char_rnn(submission, score, email, token):
grader = grading.Grader("cULEpp2NEeemQBKZKgu93A")
history, samples = submission
assert len(samples) == 25
grader.set_answer("pttMO", int(np.mean(history[:10]) > np.mean(history[-10:])))
grader.set_answer("uly0D", len(set(samples)))
grader.submit(email, token)
def submit_seq2seq(test_scores, email, token):
mean_reward = np.mean(test_scores)
print('Your average reward is {}'.format(mean_reward))
grader = grading.Grader("Xz763cqoEemssQpwd3-dGA")
grader.set_answer("GzE7v", mean_reward)
grader.submit(email, token)
def submit_breakout(agent, env, evaluate, email, token):
session_rewards = [evaluate(env, agent, n_games=1) for _ in range(100)]
mean_reward = np.mean(session_rewards)
print('Your average reward is {} over 100 episodes'.format(mean_reward))
grader = grading.Grader("WTOZHCn1EeiNwAoZNi-Hrg")
grader.set_answer("VFM7Z", mean_reward)
grader.submit(email, token)
def submit_mountain_car(generate_session, agent, email, token):
with gym.make("MountainCar-v0").env as env:
sessions = [generate_session(env, agent) for _ in range(100)]
_, _, session_rewards = zip(*sessions)
session_rewards = np.array(session_rewards)
grader = grading.Grader("EyYJW9bUEeeXyQ5ZPWKHGg")
grader.set_answer("mXDUE", np.mean(session_rewards))
grader.submit(email, token)
def submit_cartpole(generate_session, email, token):
sessions = [generate_session() for _ in range(100)]
# Bug: See https://www.coursera.org/learn/practical-rl/programming/L47Fs/approximate-qlearning-cartpole/discussions/threads/K9-K8lblEeiw7g60zbwcSA
# session_rewards, _, _ = map(np.array, zip(*sessions))
session_rewards = np.array(sessions)
grader = grading.Grader("RDofv-QXEeeaGw6kpIOf3g")
grader.set_answer("NRNkl", int(np.mean(session_rewards)))
grader.submit(email, token)
def submit_cartpole(generate_session, email, token):
with gym.make("CartPole-v0").env as env:
session_rewards = [generate_session(env) for _ in range(100)]
mean_reward = np.mean(session_rewards)
print('Your average reward is {} over 100 episodes'.format(mean_reward))
grader = grading.Grader("RDofv-QXEeeaGw6kpIOf3g")
grader.set_answer("NRNkl", mean_reward)
grader.submit(email, token)
def submit_breakout(agent, env, evaluate, email, token):
sessions = [evaluate(env, agent, n_games=1) for _ in range(100)]
session_rewards = np.array(sessions)
grader = grading.Grader("WTOZHCn1EeiNwAoZNi-Hrg")
grader.set_answer("VFM7Z", int(np.mean(session_rewards)))
grader.submit(email, token)
####session_rewards, _, _ = map(np.array, zip(*sessions))
def submit_qlearning_all(rewards_q1, rewards_q2, email, token):
grader = grading.Grader("XbjcGd7xEeeDzRKutDCmyA")
flag1 = np.mean(rewards_q1[-10:])
grader.set_answer("5NB4z", flag1)
flag2 = np.mean(rewards_q2[-10:])
grader.set_answer("CkyJ4", flag2)
grader.submit(email, token)
def submit_cartpole_pytorch(generate_session, email, token):
env = gym.make("CartPole-v0")
if hasattr(env, '_max_episode_steps'):
env = env.env
sessions = [np.sum(generate_session(env)[2]) for _ in range(100)]
session_rewards = np.array(sessions)
grader = grading.Grader("oyT3Bt7yEeeQvhJmhysb5g")
grader.set_answer("7QKmA", int(np.mean(session_rewards)))
grader.submit(email, token)
def submit_cartpole(generate_session, email, token):
with gym.make("CartPole-v0").env as env:
session_rewards = [
np.sum(generate_session(env)[2]) for _ in range(100)
]
mean_reward = np.mean(session_rewards)
print('Your average reward is {} over 100 episodes'.format(mean_reward))
grader = grading.Grader("oyT3Bt7yEeeQvhJmhysb5g")
grader.set_answer("7QKmA", mean_reward)
grader.submit(email, token)
def submit_kungfu(agent, env, evaluate, email, token):
with tqdm.trange(10, desc='Evaluating your agent') as progress_bar:
session_rewards = [
evaluate(agent=agent, env=env, n_games=1) for _ in progress_bar
]
mean_reward = np.mean(session_rewards)
print('Your average reward is {} over 10 episodes'.format(mean_reward))
grader = grading.Grader("6sPnVCn6EeieSRL7rCBNJA")
grader.set_answer("HhNVX", mean_reward)
grader.submit(email, token)
def submit_experience_replay(rewards_replay, rewards_baseline, email, token):
flag1 = np.mean(rewards_replay[:100]) - np.mean(rewards_baseline[:100])
flag2 = np.mean(rewards_replay[-100:])
flag3 = np.mean(rewards_baseline[-100:])
grader = grading.Grader("XUt-8d7yEee8nwq8KJgXXg")
grader.set_answer("iEQwT", flag1)
grader.set_answer("8N1Wm", flag2)
grader.set_answer("F0Am8", flag3)
grader.submit(email, token)
def submit_sarsa(rewards_ql, rewards_sarsa, email, token):
flag1 = np.mean(rewards_ql[-100:])
flag2 = np.mean(rewards_sarsa[-100:])
flag3 = np.mean(rewards_sarsa[-100:]) - np.mean(rewards_ql[-100:])
grader = grading.Grader("pazQX97xEee_JA6t1Myltg")
grader.set_answer("ZarWJ", flag1)
grader.set_answer("izJi4", flag2)
grader.set_answer("frgbU", flag3)
grader.submit(email, token)
def post(self):
data = flask.request.get_json()
response_id = data['response_id'].split('?')[0]
response = models.TaskResponse.query.filter_by(id=response_id).first()
course = response.task.course
if course not in current_user.get_courses_where_teacher_or_ta():
return "Permission Denied", 401
grader = grading.Grader()
correctness = grader.grade_manual_question(response_id,
data['question_id'],
data['correct'])
return json.dumps(correctness)
def test_allWrong_inDB_none_none(self):
grader = grading.Grader()
taskResponse = Mock()
self.models.TaskResponse.query.filter_by.return_value.first.return_value = taskResponse
taskResponse.graded_response = json.dumps({
'automatic_questions': [],
'manual_questions': [],
})
taskResponse.graded = None
ret = grader.calculate_correctness(1)
self.assertEqual(taskResponse.graded, True)
self.assertEqual(ret, 0)
def test_ABHIJKMNHIJKMNHPQS(self):
grader = grading.Grader()
taskResponse = Mock()
self.models.TaskResponse.query.filter_by.return_value.first.return_value = taskResponse
taskResponse.graded_response = json.dumps({
'automatic_questions': [],
'manual_questions': [{}]
})
taskResponse.graded = None
ret = grader.calculate_correctness(1)
self.assertEqual(taskResponse.graded, False)
self.assertEqual(ret, 0)
def post(self):
data = flask.request.get_json()
response_id = data['response_id'].split('?')[0]
response = models.TaskResponse.query.filter_by(id=response_id).first()
course = response.task.course
if course not in current_user.get_courses_where_teacher_or_ta():
return "Permission Denied", 401
grader = grading.Grader()
grader.grade_manual_question(response_id,
data['question_id'],
data['critical'],
category="critical")
models.db.session.commit()
return ""
def test_mixed_false_none_positive(self):
grader = grading.Grader()
taskResponse = Mock()
self.models.TaskResponse.query.filter_by.return_value.first.return_value = None
taskResponse.graded_response = json.dumps({
'automatic_questions': [],
'manual_questions': [{
'correctness': 50
}],
})
taskResponse.graded = None
ret = grader.calculate_correctness(1)
self.assertEqual(taskResponse.graded, None)
self.assertEqual(ret, 0)
def test_ABCEFMPSUV(self):
grader = grading.Grader()
taskResponse = Mock()
taskResponse.graded_response = False
taskResponse.response = json.dumps({
'automatic_questions': [],
'manual_questions': []
})
taskResponse.correctness_grade = None
taskResponse.task = Mock()
taskResponse.task.questions = json.dumps([])
self.models.TaskResponse.query.filter_by.return_value.first.return_value = taskResponse
ret = grader.grade_automatic_questions(1)
self.assertEqual(taskResponse.correctness_grade, 0)
self.assertEqual(ret, [])
def test_correctness(self):
grader = grading.Grader()
grader.calculate_correctness = Mock()
grader.calculate_correctness.return_value = "grade"
grader.grade_manual_question("notused", 2, False)
exp = {
'manual_questions': [{
'questionID': 1
}, {
'questionID': 2,
'correctness': False
}]
}
self.assertEqual(self.response.correctness_grade, "grade")
self.assertEqual(self.response.graded_response, json.dumps(exp))