def post(self):
# Sort questions into a dictionary based on their unit number
questions_by_usage_id = event_transforms.get_questions_by_usage_id(
self.app_context)
sorted_questions_by_unit = self._rearrange_dict_by_field(
questions_by_usage_id, 'unit')
# Only use Students we generated.
students = common_utils.iter_all(models.Student.all().filter(
'email >', 'gen_sample_student_').filter('email <',
'gen_sample_student`'))
source = 'submit-assessment'
for student in students:
user = users.User(email=student.email, _user_id=student.user_id)
assessment_data = self._generate_answers(student,
sorted_questions_by_unit)
for data in assessment_data:
EventEntity.record(
source, user,
transforms.dumps({
'values': data,
'location': 'AnswerHandler'
}))
self.redirect(self.request.referer)
def main(self):
# By the time main() is invoked, arguments are parsed and available as
# self.args. If you need more complicated argument validation than
# argparse gives you, do it here:
if self.args.batch_size < 1:
sys.exit('--batch size must be positive')
if os.path.exists(self.args.path):
sys.exit('Cannot download to %s; file exists' % self.args.path)
# Arguments passed to etl.py are also parsed and available as
# self.etl_args. Here we use them to figure out the requested course's
# namespace.
namespace = etl_lib.get_context(
self.etl_args.course_url_prefix).get_namespace_name()
# Because our models are namespaced, we need to change to the requested
# course's namespace when doing datastore reads.
with common_utils.Namespace(namespace):
# This base query can be modified to add whatever filters you need.
query = models.Student.all()
students = common_utils.iter_all(query, self.args.batch_size)
# Write the results. Done!
with open(self.args.path, 'w') as f:
for student in students:
f.write(student.email)
f.write('\n')
def main(self):
# By the time main() is invoked, arguments are parsed and available as
# self.args. If you need more complicated argument validation than
# argparse gives you, do it here:
if self.args.batch_size < 1:
sys.exit('--batch size must be positive')
if os.path.exists(self.args.path):
sys.exit('Cannot download to %s; file exists' % self.args.path)
# Arguments passed to etl.py are also parsed and available as
# self.etl_args. Here we use them to figure out the requested course's
# namespace.
namespace = etl_lib.get_context(
self.etl_args.course_url_prefix).get_namespace_name()
# Because our models are namespaced, we need to change to the requested
# course's namespace when doing datastore reads.
with common_utils.Namespace(namespace):
# This base query can be modified to add whatever filters you need.
query = models.Student.all()
students = common_utils.iter_all(query, self.args.batch_size)
# Write the results. Done!
with open(self.args.path, 'w') as f:
for student in students:
f.write(student.email)
f.write('\n')
def _get_question_texts(self):
"""Returns a dictionary with the question text for each question ID."""
question_texts = {}
for question in common_utils.iter_all(models.QuestionEntity.all()):
question_id = str(question.key().id())
question_data = transforms.loads(question.data)
question_texts[question_id] = question_data['question']
return question_texts
def _get_question_texts(self):
"""Returns a dictionary with the question text for each question ID."""
question_texts = {}
for question in common_utils.iter_all(models.QuestionEntity.all()):
question_id = str(question.key().id())
question_data = transforms.loads(question.data)
question_texts[question_id] = question_data['question']
return question_texts
def load_all(cls):
"""Loads all DTOs that have valid entities, in no particular order.
Returns:
An iterator that produces cls.DTOs created from all the ENTITY
values in the Datastore, in no particular order.
"""
with common_utils.Namespace(appengine_config.DEFAULT_NAMESPACE_NAME):
for e in common_utils.iter_all(cls.ENTITY.all()):
if e.key().name():
yield cls.new_dto('', entity=e)
def load_all(cls):
"""Loads all DTOs that have valid entities, in no particular order.
Returns:
An iterator that produces cls.DTOs created from all the ENTITY
values in the Datastore, in no particular order.
"""
with common_utils.Namespace(appengine_config.DEFAULT_NAMESPACE_NAME):
for e in common_utils.iter_all(cls.ENTITY.all()):
if e.key().name():
yield cls.new_dto('', entity=e)
def get_announcements(cls, locale=None):
memcache_key = cls._cache_key(locale)
items = models.MemcacheManager.get(memcache_key)
if items is None:
items = list(common_utils.iter_all(AnnouncementEntity.all()))
items.sort(key=lambda item: item.date, reverse=True)
if locale:
cls._translate_content(items)
# TODO(psimakov): prepare to exceed 1MB max item size
# read more here: http://stackoverflow.com
# /questions/5081502/memcache-1-mb-limit-in-google-app-engine
models.MemcacheManager.set(memcache_key, items)
return items
def _rearrange_dict_by_field(self, old_dict, sorted_field):
"""Rearranges and filters a dictionary of questions.
Takes a dictionary of entries of the form
{id1 : { 'val1': _, 'val2': _ }, id2 : { 'val1': _, 'val2': _ }, ...}
and rearranges it so that items that match for the chosen field are
placed.
When we arrange by unit number, the output will be:
{ <unit_num_1> : <dictionary of questions from unit_num_1>,
<unit_num_2> : <dictionary of questions from unit_num_2>, ...}
We also include only the questions whose text begins with the correct
prefix marking it as an automatically generated questions.
"""
# First we need to get the set of ID's for automatically generated
# questions, and their graders.
question_entities = common_utils.iter_all(models.QuestionEntity.all())
grader_dict = {}
auto_generated_ids = set()
for question_entity in question_entities:
question_data = transforms.loads(question_entity.data)
question_id = str(question_entity.key().id())
text = question_data['question']
if text.startswith(GenerateSampleQuizHandler.QUESTION_PREFIX):
auto_generated_ids.add(question_id)
grader_dict[question_id] = question_data['graders']
sorted_dict = {}
for instance_id in old_dict:
old_entry = old_dict[instance_id]
question_id = old_entry['id']
if question_id in auto_generated_ids:
sort_val = old_entry[sorted_field]
if sort_val in sorted_dict:
sorted_dict[sort_val][instance_id] = old_dict[instance_id]
else:
sorted_dict[sort_val] = {
instance_id: old_dict[instance_id]
}
grader = grader_dict[question_id]
sorted_dict[sort_val][instance_id]['graders'] = grader
return sorted_dict
def _rearrange_dict_by_field(self, old_dict, sorted_field):
"""Rearranges and filters a dictionary of questions.
Takes a dictionary of entries of the form
{id1 : { 'val1': _, 'val2': _ }, id2 : { 'val1': _, 'val2': _ }, ...}
and rearranges it so that items that match for the chosen field are
placed.
When we arrange by unit number, the output will be:
{ <unit_num_1> : <dictionary of questions from unit_num_1>,
<unit_num_2> : <dictionary of questions from unit_num_2>, ...}
We also include only the questions whose text begins with the correct
prefix marking it as an automatically generated questions.
"""
# First we need to get the set of ID's for automatically generated
# questions, and their graders.
question_entities = common_utils.iter_all(models.QuestionEntity.all())
grader_dict = {}
auto_generated_ids = set()
for question_entity in question_entities:
question_data = transforms.loads(question_entity.data)
question_id = str(question_entity.key().id())
text = question_data['question']
if text.startswith(GenerateSampleQuizHandler.QUESTION_PREFIX):
auto_generated_ids.add(question_id)
grader_dict[question_id] = question_data['graders']
sorted_dict = {}
for instance_id in old_dict:
old_entry = old_dict[instance_id]
question_id = old_entry['id']
if question_id in auto_generated_ids:
sort_val = old_entry[sorted_field]
if sort_val in sorted_dict:
sorted_dict[sort_val][instance_id] = old_dict[instance_id]
else:
sorted_dict[sort_val] = {instance_id:
old_dict[instance_id]}
grader = grader_dict[question_id]
sorted_dict[sort_val][instance_id]['graders'] = grader
return sorted_dict
def _compare_answers(self, answers_before, answers_after):
# We need a dictionary of the text for each question ID, so we can check
# whether a question has been marked as automatically generated.
question_names = self._get_question_texts()
all_questions = event_transforms.get_questions_by_usage_id(
self.app_context)
for student in common_utils.iter_all(models.Student.all()):
user_id = str(student.user_id)
if not self._is_generated_student(student):
# If a student has not been automatically generated, then
# either they should not exist in either dictionary, or
# their number of answers provided should be the same for
# every question that they have answered
if user_id not in answers_before:
self.assertEquals(True, user_id not in answers_after)
else:
for instance_id in all_questions:
before = answers_before[user_id][instance_id]
after = answers_after[user_id][instance_id]
self.assertEquals(before, after)
else:
# If a student has been automatically generated, then their
# number of answers should have increased by one for every
# automatically generated question.
for instance_id in all_questions:
countBefore = 0
countAfter = 0
if user_id in answers_before:
countBefore = answers_before[user_id][instance_id]
if user_id in answers_after:
countAfter = answers_after[user_id][instance_id]
question_id = all_questions[instance_id]['id']
if question_names[question_id].startswith('gen_sample:'):
self.assertEquals(countBefore + 1, countAfter)
else:
self.assertEquals(countBefore, countAfter)
def _compare_answers(self, answers_before, answers_after):
# We need a dictionary of the text for each question ID, so we can check
# whether a question has been marked as automatically generated.
question_names = self._get_question_texts()
all_questions = event_transforms.get_questions_by_usage_id(
self.app_context)
for student in common_utils.iter_all(models.Student.all()):
user_id = str(student.user_id)
if not self._is_generated_student(student):
# If a student has not been automatically generated, then
# either they should not exist in either dictionary, or
# their number of answers provided should be the same for
# every question that they have answered
if user_id not in answers_before:
self.assertEquals(True, user_id not in answers_after)
else:
for instance_id in all_questions:
before = answers_before[user_id][instance_id]
after = answers_after[user_id][instance_id]
self.assertEquals(before, after)
else:
# If a student has been automatically generated, then their
# number of answers should have increased by one for every
# automatically generated question.
for instance_id in all_questions:
countBefore = 0
countAfter = 0
if user_id in answers_before:
countBefore = answers_before[user_id][instance_id]
if user_id in answers_after:
countAfter = answers_after[user_id][instance_id]
question_id = all_questions[instance_id]['id']
if question_names[question_id].startswith('gen_sample:'):
self.assertEquals(countBefore + 1, countAfter)
else:
self.assertEquals(countBefore, countAfter)
def post(self):
# Sort questions into a dictionary based on their unit number
questions_by_usage_id = event_transforms.get_questions_by_usage_id(
self.app_context)
sorted_questions_by_unit = self._rearrange_dict_by_field(
questions_by_usage_id, 'unit')
# Only use Students we generated.
students = common_utils.iter_all(models.Student.all().filter(
'email >', 'gen_sample_student_').filter(
'email <', 'gen_sample_student`'))
source = 'submit-assessment'
for student in students:
user = users.User(email=student.email, _user_id=student.user_id)
assessment_data = self._generate_answers(student,
sorted_questions_by_unit)
for data in assessment_data:
EventEntity.record(source, user, transforms.dumps({
'values': data, 'location': 'AnswerHandler'}))
self.redirect(self.request.referer)
def _get_answers(self):
"""Generates a dictionary of answer counts.
For each student in the course, we create a dictionary in which the
key is the question instance id and the value is a list of answers that
student has given. Then we put all of those dictionaries as entries into
a new dictionary indexed by the student ID.
"""
answer_dict = {}
for event in common_utils.iter_all(models.EventEntity.all()):
if event.source == 'submit-assessment':
if event.user_id not in answer_dict:
# If we attempt to get a question count when a user has
# not answered a question, this will automatically create
# a new entry that is equal to zero, instead of raising a
# KeyError.
answer_dict[event.user_id] = collections.defaultdict(int)
data = transforms.loads(event.data)
for instance_id in data['values']['containedTypes']:
answer_dict[event.user_id][instance_id] += 1
return answer_dict
def _get_answers(self):
"""Generates a dictionary of answer counts.
For each student in the course, we create a dictionary in which the
key is the question instance id and the value is a list of answers that
student has given. Then we put all of those dictionaries as entries into
a new dictionary indexed by the student ID.
"""
answer_dict = {}
for event in common_utils.iter_all(models.EventEntity.all()):
if event.source == 'submit-assessment':
if event.user_id not in answer_dict:
# If we attempt to get a question count when a user has
# not answered a question, this will automatically create
# a new entry that is equal to zero, instead of raising a
# KeyError.
answer_dict[event.user_id] = collections.defaultdict(int)
data = transforms.loads(event.data)
for instance_id in data['values']['containedTypes']:
answer_dict[event.user_id][instance_id] += 1
return answer_dict
def get_any_undeleted_kind_name(cls):
for kind in common_utils.iter_all(metadata.Kind.all()):
kind_name = kind.kind_name
if not cls.IGNORE_KINDS.match(kind_name):
return kind_name
return None
def get_any_undeleted_kind_name(cls):
for kind in common_utils.iter_all(metadata.Kind.all()):
kind_name = kind.kind_name
if not cls.IGNORE_KINDS.match(kind_name):
return kind_name
return None
def _get_student_ids(self):
students = common_utils.iter_all(models.Student.all())
return [student.user_id for student in students]
def _get_student_ids(self):
students = common_utils.iter_all(models.Student.all())
return [student.user_id for student in students]
