def fix_studentmodules_in_list(self, save_changes, idlist_path):
'''Read in the list of StudentModule objects that might need fixing, and then fix each one'''
# open file and read id values from it:
for line in open(idlist_path, 'r'):
student_module_id = line.strip()
# skip the header, if present:
if student_module_id == 'id':
continue
try:
module = StudentModule.objects.select_related('student').get(id=student_module_id)
except StudentModule.DoesNotExist:
LOG.error(u"Unable to find student module with id = %s: skipping... ", student_module_id)
continue
self.remove_studentmodule_input_state(module, save_changes)
user_state_client = DjangoXBlockUserStateClient()
hist_modules = user_state_client.get_history(module.student.username, module.module_state_key)
for hist_module in hist_modules:
self.remove_studentmodulehistory_input_state(hist_module, save_changes)
if self.num_visited % 1000 == 0:
LOG.info(" Progress: updated {0} of {1} student modules".format(self.num_changed, self.num_visited))
LOG.info(" Progress: updated {0} of {1} student history modules".format(self.num_hist_changed,
self.num_hist_visited))
def fix_studentmodules_in_list(self, save_changes, idlist_path):
'''Read in the list of StudentModule objects that might need fixing, and then fix each one'''
# open file and read id values from it:
for line in open(idlist_path, 'r'):
student_module_id = line.strip()
# skip the header, if present:
if student_module_id == 'id':
continue
try:
module = StudentModule.objects.select_related('student').get(id=student_module_id)
except StudentModule.DoesNotExist:
LOG.error(u"Unable to find student module with id = %s: skipping... ", student_module_id)
continue
self.remove_studentmodule_input_state(module, save_changes)
user_state_client = DjangoXBlockUserStateClient()
hist_modules = user_state_client.get_history(module.student.username, module.module_state_key)
for hist_module in hist_modules:
self.remove_studentmodulehistory_input_state(hist_module, save_changes)
if self.num_visited % 1000 == 0:
LOG.info(" Progress: updated %s of %s student modules", self.num_changed, self.num_visited)
LOG.info(
" Progress: updated %s of %s student history modules",
self.num_hist_changed,
self.num_hist_visited
)
def test_submission_history_contents(self):
# log into a staff account
admin = AdminFactory.create()
self.client.login(username=admin.username, password='******')
usage_key = self.course_key.make_usage_key('problem', 'test-history')
state_client = DjangoXBlockUserStateClient(admin)
# store state via the UserStateClient
state_client.set(
username=admin.username,
block_key=usage_key,
state={'field_a': 'x', 'field_b': 'y'}
)
set_score(admin.id, usage_key, 0, 3)
state_client.set(
username=admin.username,
block_key=usage_key,
state={'field_a': 'a', 'field_b': 'b'}
)
set_score(admin.id, usage_key, 3, 3)
url = reverse('submission_history', kwargs={
'course_id': unicode(self.course_key),
'student_username': admin.username,
'location': unicode(usage_key),
})
response = self.client.get(url)
response_content = HTMLParser().unescape(response.content)
# We have update the state 4 times: twice to change content, and twice
# to set the scores. We'll check that the identifying content from each is
# displayed (but not the order), and also the indexes assigned in the output
# #1 - #4
self.assertIn('#1', response_content)
self.assertIn(json.dumps({'field_a': 'a', 'field_b': 'b'}, sort_keys=True, indent=2), response_content)
self.assertIn("Score: 0.0 / 3.0", response_content)
self.assertIn(json.dumps({'field_a': 'x', 'field_b': 'y'}, sort_keys=True, indent=2), response_content)
self.assertIn("Score: 3.0 / 3.0", response_content)
self.assertIn('#4', response_content)
def _build_student_data(cls, user_id, course_key, usage_key_str):
"""
Generate a list of problem responses for all problem under the
``problem_location`` root.
Arguments:
user_id (int): The user id for the user generating the report
course_key (CourseKey): The ``CourseKey`` for the course whose report
is being generated
usage_key_str (str): The generated report will include this
block and it child blocks.
Returns:
Tuple[List[Dict], List[str]]: Returns a list of dictionaries
containing the student data which will be included in the
final csv, and the features/keys to include in that CSV.
"""
usage_key = UsageKey.from_string(usage_key_str).map_into_course(course_key)
user = get_user_model().objects.get(pk=user_id)
course_blocks = get_course_blocks(user, usage_key)
student_data = []
max_count = settings.FEATURES.get('MAX_PROBLEM_RESPONSES_COUNT')
store = modulestore()
user_state_client = DjangoXBlockUserStateClient()
student_data_keys = set()
with store.bulk_operations(course_key):
for title, path, block_key in cls._build_problem_list(course_blocks, usage_key):
# Chapter and sequential blocks are filtered out since they include state
# which isn't useful for this report.
if block_key.block_type in ('sequential', 'chapter'):
continue
block = store.get_item(block_key)
generated_report_data = {}
# Blocks can implement the generate_report_data method to provide their own
# human-readable formatting for user state.
if hasattr(block, 'generate_report_data'):
try:
user_state_iterator = user_state_client.iter_all_for_block(block_key)
generated_report_data = {
username: state
for username, state in
block.generate_report_data(user_state_iterator, max_count)
}
except NotImplementedError:
pass
responses = list_problem_responses(course_key, block_key, max_count)
student_data += responses
for response in responses:
response['title'] = title
# A human-readable location for the current block
response['location'] = ' > '.join(path)
# A machine-friendly location for the current block
response['block_key'] = str(block_key)
user_data = generated_report_data.get(response['username'], {})
response.update(user_data)
student_data_keys = student_data_keys.union(user_data.keys())
if max_count is not None:
max_count -= len(responses)
if max_count <= 0:
break
# Keep the keys in a useful order, starting with username, title and location,
# then the columns returned by the xblock report generator in sorted order and
# finally end with the more machine friendly block_key and state.
student_data_keys_list = (
['username', 'title', 'location'] +
sorted(student_data_keys) +
['block_key', 'state']
)
return student_data, student_data_keys_list
def __init__(self, user, course_id):
self._cache = defaultdict(dict)
self.course_id = course_id
self.user = user
self._client = DjangoXBlockUserStateClient(self.user)
class UserStateCache(object):
"""
Cache for Scope.user_state xblock field data.
"""
def __init__(self, user, course_id):
self._cache = defaultdict(dict)
self.course_id = course_id
self.user = user
self._client = DjangoXBlockUserStateClient(self.user)
def cache_fields(self, fields, xblocks, aside_types): # pylint: disable=unused-argument
"""
Load all fields specified by ``fields`` for the supplied ``xblocks``
and ``aside_types`` into this cache.
Arguments:
fields (list of str): Field names to cache.
xblocks (list of :class:`XBlock`): XBlocks to cache fields for.
aside_types (list of str): Aside types to cache fields for.
"""
block_field_state = self._client.get_many(
self.user.username,
_all_usage_keys(xblocks, aside_types),
)
for usage_key, field_state in block_field_state:
self._cache[usage_key] = field_state
@contract(kvs_key=DjangoKeyValueStore.Key)
def set(self, kvs_key, value):
"""
Set the specified `kvs_key` to the field value `value`.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
value: The field value to store
"""
self.set_many({kvs_key: value})
@contract(kvs_key=DjangoKeyValueStore.Key, returns="datetime|None")
def last_modified(self, kvs_key):
"""
Return when the supplied field was changed.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The key representing the cached field
Returns: datetime if there was a modified date, or None otherwise
"""
return self._client.get_mod_date(
self.user.username,
kvs_key.block_scope_id,
fields=[kvs_key.field_name],
).get(kvs_key.field_name)
@contract(kv_dict="dict(DjangoKeyValueStore_Key: *)")
def set_many(self, kv_dict):
"""
Set the specified fields to the supplied values.
Arguments:
kv_dict (dict): A dictionary mapping :class:`~DjangoKeyValueStore.Key`
objects to values to set.
"""
pending_updates = defaultdict(dict)
for kvs_key, value in kv_dict.items():
cache_key = self._cache_key_for_kvs_key(kvs_key)
pending_updates[cache_key][kvs_key.field_name] = value
try:
self._client.set_many(self.user.username, pending_updates)
except DatabaseError:
raise KeyValueMultiSaveError([])
finally:
self._cache.update(pending_updates)
@contract(kvs_key=DjangoKeyValueStore.Key)
def get(self, kvs_key):
"""
Return the django model object specified by `kvs_key` from
the cache.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Returns: A django orm object from the cache
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
if cache_key not in self._cache:
raise KeyError(kvs_key.field_name)
return self._cache[cache_key][kvs_key.field_name]
@contract(kvs_key=DjangoKeyValueStore.Key)
def delete(self, kvs_key):
"""
Delete the value specified by `kvs_key`.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Raises: KeyError if key isn't found in the cache
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
if cache_key not in self._cache:
raise KeyError(kvs_key.field_name)
field_state = self._cache[cache_key]
if kvs_key.field_name not in field_state:
raise KeyError(kvs_key.field_name)
self._client.delete(self.user.username,
cache_key,
fields=[kvs_key.field_name])
del field_state[kvs_key.field_name]
@contract(kvs_key=DjangoKeyValueStore.Key, returns=bool)
def has(self, kvs_key):
"""
Return whether the specified `kvs_key` is set.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Returns: bool
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
return (cache_key in self._cache
and kvs_key.field_name in self._cache[cache_key])
@contract(user_id=int,
usage_key=UsageKey,
score="number|None",
max_score="number|None")
def set_score(self, user_id, usage_key, score, max_score):
"""
UNSUPPORTED METHOD
Set the score and max_score for the specified user and xblock usage.
"""
student_module, created = StudentModule.objects.get_or_create(
student_id=user_id,
module_state_key=usage_key,
course_id=usage_key.course_key,
defaults={
'grade': score,
'max_grade': max_score,
})
if not created:
student_module.grade = score
student_module.max_grade = max_score
student_module.save()
def __len__(self):
return len(self._cache)
def _cache_key_for_kvs_key(self, key):
"""
Return the key used in this DjangoOrmFieldCache for the specified KeyValueStore key.
Arguments:
key (:class:`~DjangoKeyValueStore.Key`): The key representing the cached field
"""
return key.block_scope_id
def __init__(self, user, course_id):
self._cache = defaultdict(dict)
self.course_id = course_id
self.user = user
self._client = DjangoXBlockUserStateClient(self.user)
class UserStateCache(object):
"""
Cache for Scope.user_state xblock field data.
"""
def __init__(self, user, course_id):
self._cache = defaultdict(dict)
self.course_id = course_id
self.user = user
self._client = DjangoXBlockUserStateClient(self.user)
def cache_fields(self, fields, xblocks, aside_types): # pylint: disable=unused-argument
"""
Load all fields specified by ``fields`` for the supplied ``xblocks``
and ``aside_types`` into this cache.
Arguments:
fields (list of str): Field names to cache.
xblocks (list of :class:`XBlock`): XBlocks to cache fields for.
aside_types (list of str): Aside types to cache fields for.
"""
block_field_state = self._client.get_many(
self.user.username,
_all_usage_keys(xblocks, aside_types),
)
for user_state in block_field_state:
self._cache[user_state.block_key] = user_state.state
@contract(kvs_key=DjangoKeyValueStore.Key)
def set(self, kvs_key, value):
"""
Set the specified `kvs_key` to the field value `value`.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
value: The field value to store
"""
self.set_many({kvs_key: value})
@contract(kvs_key=DjangoKeyValueStore.Key, returns="datetime|None")
def last_modified(self, kvs_key):
"""
Return when the supplied field was changed.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The key representing the cached field
Returns: datetime if there was a modified date, or None otherwise
"""
try:
return self._client.get(
self.user.username,
kvs_key.block_scope_id,
fields=[kvs_key.field_name],
).updated
except self._client.DoesNotExist:
return None
@contract(kv_dict="dict(DjangoKeyValueStore_Key: *)")
def set_many(self, kv_dict):
"""
Set the specified fields to the supplied values.
Arguments:
kv_dict (dict): A dictionary mapping :class:`~DjangoKeyValueStore.Key`
objects to values to set.
"""
pending_updates = defaultdict(dict)
for kvs_key, value in kv_dict.items():
cache_key = self._cache_key_for_kvs_key(kvs_key)
pending_updates[cache_key][kvs_key.field_name] = value
try:
self._client.set_many(
self.user.username,
pending_updates
)
except DatabaseError:
log.exception("Saving user state failed for %s", self.user.username)
raise KeyValueMultiSaveError([])
finally:
self._cache.update(pending_updates)
@contract(kvs_key=DjangoKeyValueStore.Key)
def get(self, kvs_key):
"""
Return the django model object specified by `kvs_key` from
the cache.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Returns: A django orm object from the cache
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
if cache_key not in self._cache:
raise KeyError(kvs_key.field_name)
return self._cache[cache_key][kvs_key.field_name]
@contract(kvs_key=DjangoKeyValueStore.Key)
def delete(self, kvs_key):
"""
Delete the value specified by `kvs_key`.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Raises: KeyError if key isn't found in the cache
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
if cache_key not in self._cache:
raise KeyError(kvs_key.field_name)
field_state = self._cache[cache_key]
if kvs_key.field_name not in field_state:
raise KeyError(kvs_key.field_name)
self._client.delete(self.user.username, cache_key, fields=[kvs_key.field_name])
del field_state[kvs_key.field_name]
@contract(kvs_key=DjangoKeyValueStore.Key, returns=bool)
def has(self, kvs_key):
"""
Return whether the specified `kvs_key` is set.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Returns: bool
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
return (
cache_key in self._cache and
kvs_key.field_name in self._cache[cache_key]
)
def __len__(self):
return len(self._cache)
def _cache_key_for_kvs_key(self, key):
"""
Return the key used in this DjangoOrmFieldCache for the specified KeyValueStore key.
Arguments:
key (:class:`~DjangoKeyValueStore.Key`): The key representing the cached field
"""
return key.block_scope_id
def setUp(self):
super(TestDjangoUserStateClient, self).setUp()
self.client = DjangoXBlockUserStateClient()
self.users = defaultdict(UserFactory.create)
def _build_student_data(
cls,
user_id,
course_key,
usage_key_str_list,
filter_types=None,
):
"""
Generate a list of problem responses for all problem under the
``problem_location`` root.
Arguments:
user_id (int): The user id for the user generating the report
course_key (CourseKey): The ``CourseKey`` for the course whose report
is being generated
usage_key_str_list (List[str]): The generated report will include these
blocks and their child blocks.
filter_types (List[str]): The report generator will only include data for
block types in this list.
Returns:
Tuple[List[Dict], List[str]]: Returns a list of dictionaries
containing the student data which will be included in the
final csv, and the features/keys to include in that CSV.
"""
usage_keys = [
UsageKey.from_string(usage_key_str).map_into_course(course_key)
for usage_key_str in usage_key_str_list
]
user = get_user_model().objects.get(pk=user_id)
student_data = []
max_count = settings.FEATURES.get('MAX_PROBLEM_RESPONSES_COUNT')
store = modulestore()
user_state_client = DjangoXBlockUserStateClient()
student_data_keys = set()
with store.bulk_operations(course_key):
for usage_key in usage_keys:
if max_count is not None and max_count <= 0:
break
course_blocks = get_course_blocks(user, usage_key)
base_path = cls._build_block_base_path(
store.get_item(usage_key))
for title, path, block_key in cls._build_problem_list(
course_blocks, usage_key):
# Chapter and sequential blocks are filtered out since they include state
# which isn't useful for this report.
if block_key.block_type in ('sequential', 'chapter'):
continue
if filter_types is not None and block_key.block_type not in filter_types:
continue
block = store.get_item(block_key)
generated_report_data = defaultdict(list)
# Blocks can implement the generate_report_data method to provide their own
# human-readable formatting for user state.
if hasattr(block, 'generate_report_data'):
try:
user_state_iterator = user_state_client.iter_all_for_block(
block_key)
for username, state in block.generate_report_data(
user_state_iterator, max_count):
generated_report_data[username].append(state)
except NotImplementedError:
pass
responses = []
for response in list_problem_responses(
course_key, block_key, max_count):
response['title'] = title
# A human-readable location for the current block
response['location'] = ' > '.join(base_path + path)
# A machine-friendly location for the current block
response['block_key'] = str(block_key)
# A block that has a single state per user can contain multiple responses
# within the same state.
user_states = generated_report_data.get(
response['username'], [])
if user_states:
# For each response in the block, copy over the basic data like the
# title, location, block_key and state, and add in the responses
for user_state in user_states:
user_response = response.copy()
user_response.update(user_state)
student_data_keys = student_data_keys.union(
user_state.keys())
responses.append(user_response)
else:
responses.append(response)
student_data += responses
if max_count is not None:
max_count -= len(responses)
if max_count <= 0:
break
# Keep the keys in a useful order, starting with username, title and location,
# then the columns returned by the xblock report generator in sorted order and
# finally end with the more machine friendly block_key and state.
student_data_keys_list = (['username', 'title', 'location'] +
sorted(student_data_keys) +
['block_key', 'state'])
return student_data, student_data_keys_list
def _build_student_data(cls, user_id, course_key, usage_key_str):
"""
Generate a list of problem responses for all problem under the
``problem_location`` root.
Arguments:
user_id (int): The user id for the user generating the report
course_key (CourseKey): The ``CourseKey`` for the course whose report
is being generated
usage_key_str (str): The generated report will include this
block and it child blocks.
Returns:
List[Dict]: Returns a list of dictionaries containing the student
data which will be included in the final csv.
"""
usage_key = UsageKey.from_string(usage_key_str).map_into_course(
course_key)
user = get_user_model().objects.get(pk=user_id)
course_blocks = get_course_blocks(user, usage_key)
student_data = []
max_count = settings.FEATURES.get('MAX_PROBLEM_RESPONSES_COUNT')
store = modulestore()
user_state_client = DjangoXBlockUserStateClient()
with store.bulk_operations(course_key):
for title, path, block_key in cls._build_problem_list(
course_blocks, usage_key):
# Chapter and sequential blocks are filtered out since they include state
# which isn't useful for this report.
if block_key.block_type in ('sequential', 'chapter'):
continue
block = store.get_item(block_key)
# Blocks can implement the generate_report_data method to provide their own
# human-readable formatting for user state.
if hasattr(block, 'generate_report_data'):
try:
user_state_iterator = user_state_client.iter_all_for_block(
block_key)
responses = [{
'username': username,
'state': state
} for username, state in block.generate_report_data(
user_state_iterator, max_count)]
except NotImplementedError:
responses = list_problem_responses(
course_key, block_key, max_count)
else:
responses = list_problem_responses(course_key, block_key,
max_count)
student_data += responses
for response in responses:
response['title'] = title
# A human-readable location for the current block
response['location'] = ' > '.join(path)
# A machine-friendly location for the current block
response['block_key'] = str(block_key)
if max_count is not None:
max_count -= len(responses)
if max_count <= 0:
break
return student_data
class UserStateCache(object):
"""
Cache for Scope.user_state xblock field data.
"""
def __init__(self, user, course_id):
self._cache = defaultdict(dict)
self.course_id = course_id
self.user = user
self._client = DjangoXBlockUserStateClient(self.user)
def cache_fields(self, fields, xblocks, aside_types): # pylint: disable=unused-argument
"""
Load all fields specified by ``fields`` for the supplied ``xblocks``
and ``aside_types`` into this cache.
Arguments:
fields (list of str): Field names to cache.
xblocks (list of :class:`XBlock`): XBlocks to cache fields for.
aside_types (list of str): Aside types to cache fields for.
"""
block_field_state = self._client.get_many(self.user.username, _all_usage_keys(xblocks, aside_types))
for usage_key, field_state in block_field_state:
self._cache[usage_key] = field_state
@contract(kvs_key=DjangoKeyValueStore.Key)
def set(self, kvs_key, value):
"""
Set the specified `kvs_key` to the field value `value`.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
value: The field value to store
"""
self.set_many({kvs_key: value})
@contract(kvs_key=DjangoKeyValueStore.Key, returns="datetime|None")
def last_modified(self, kvs_key):
"""
Return when the supplied field was changed.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The key representing the cached field
Returns: datetime if there was a modified date, or None otherwise
"""
return self._client.get_mod_date(self.user.username, kvs_key.block_scope_id, fields=[kvs_key.field_name]).get(
kvs_key.field_name
)
@contract(kv_dict="dict(DjangoKeyValueStore_Key: *)")
def set_many(self, kv_dict):
"""
Set the specified fields to the supplied values.
Arguments:
kv_dict (dict): A dictionary mapping :class:`~DjangoKeyValueStore.Key`
objects to values to set.
"""
pending_updates = defaultdict(dict)
for kvs_key, value in kv_dict.items():
cache_key = self._cache_key_for_kvs_key(kvs_key)
pending_updates[cache_key][kvs_key.field_name] = value
try:
self._client.set_many(self.user.username, pending_updates)
except DatabaseError:
raise KeyValueMultiSaveError([])
finally:
self._cache.update(pending_updates)
@contract(kvs_key=DjangoKeyValueStore.Key)
def get(self, kvs_key):
"""
Return the django model object specified by `kvs_key` from
the cache.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Returns: A django orm object from the cache
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
if cache_key not in self._cache:
raise KeyError(kvs_key.field_name)
return self._cache[cache_key][kvs_key.field_name]
@contract(kvs_key=DjangoKeyValueStore.Key)
def delete(self, kvs_key):
"""
Delete the value specified by `kvs_key`.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Raises: KeyError if key isn't found in the cache
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
if cache_key not in self._cache:
raise KeyError(kvs_key.field_name)
field_state = self._cache[cache_key]
if kvs_key.field_name not in field_state:
raise KeyError(kvs_key.field_name)
self._client.delete(self.user.username, cache_key, fields=[kvs_key.field_name])
del field_state[kvs_key.field_name]
@contract(kvs_key=DjangoKeyValueStore.Key, returns=bool)
def has(self, kvs_key):
"""
Return whether the specified `kvs_key` is set.
Arguments:
kvs_key (`DjangoKeyValueStore.Key`): The field value to delete
Returns: bool
"""
cache_key = self._cache_key_for_kvs_key(kvs_key)
return cache_key in self._cache and kvs_key.field_name in self._cache[cache_key]
# @contract(user_id=int, usage_key=UsageKey, score="number|None", max_score="number|None")
def set_score(self, user_id, usage_key, score, max_score):
"""
UNSUPPORTED METHOD
Set the score and max_score for the specified user and xblock usage.
"""
student_module, created = StudentModule.objects.get_or_create(
student_id=user_id,
module_state_key=usage_key,
course_id=usage_key.course_key,
defaults={"grade": score, "max_grade": max_score},
)
if not created:
student_module.grade = score
student_module.max_grade = max_score
student_module.save()
def __len__(self):
return len(self._cache)
def _cache_key_for_kvs_key(self, key):
"""
Return the key used in this DjangoOrmFieldCache for the specified KeyValueStore key.
Arguments:
key (:class:`~DjangoKeyValueStore.Key`): The key representing the cached field
"""
return key.block_scope_id
