def fix_studentmodules_in_list(self, save_changes, idlist_path): '''Read in the list of StudentModule objects that might need fixing, and then fix each one''' # open file and read id values from it: for line in open(idlist_path, 'r'): student_module_id = line.strip() # skip the header, if present: if student_module_id == 'id': continue try: module = StudentModule.objects.select_related('student').get(id=student_module_id) except StudentModule.DoesNotExist: LOG.error(u"Unable to find student module with id = %s: skipping... ", student_module_id) continue self.remove_studentmodule_input_state(module, save_changes) user_state_client = DjangoXBlockUserStateClient() hist_modules = user_state_client.get_history(module.student.username, module.module_state_key) for hist_module in hist_modules: self.remove_studentmodulehistory_input_state(hist_module, save_changes) if self.num_visited % 1000 == 0: LOG.info(" Progress: updated {0} of {1} student modules".format(self.num_changed, self.num_visited)) LOG.info(" Progress: updated {0} of {1} student history modules".format(self.num_hist_changed, self.num_hist_visited))
def fix_studentmodules_in_list(self, save_changes, idlist_path): '''Read in the list of StudentModule objects that might need fixing, and then fix each one''' # open file and read id values from it: for line in open(idlist_path, 'r'): student_module_id = line.strip() # skip the header, if present: if student_module_id == 'id': continue try: module = StudentModule.objects.select_related('student').get(id=student_module_id) except StudentModule.DoesNotExist: LOG.error(u"Unable to find student module with id = %s: skipping... ", student_module_id) continue self.remove_studentmodule_input_state(module, save_changes) user_state_client = DjangoXBlockUserStateClient() hist_modules = user_state_client.get_history(module.student.username, module.module_state_key) for hist_module in hist_modules: self.remove_studentmodulehistory_input_state(hist_module, save_changes) if self.num_visited % 1000 == 0: LOG.info(" Progress: updated %s of %s student modules", self.num_changed, self.num_visited) LOG.info( " Progress: updated %s of %s student history modules", self.num_hist_changed, self.num_hist_visited )
def test_submission_history_contents(self): # log into a staff account admin = AdminFactory.create() self.client.login(username=admin.username, password='******') usage_key = self.course_key.make_usage_key('problem', 'test-history') state_client = DjangoXBlockUserStateClient(admin) # store state via the UserStateClient state_client.set( username=admin.username, block_key=usage_key, state={'field_a': 'x', 'field_b': 'y'} ) set_score(admin.id, usage_key, 0, 3) state_client.set( username=admin.username, block_key=usage_key, state={'field_a': 'a', 'field_b': 'b'} ) set_score(admin.id, usage_key, 3, 3) url = reverse('submission_history', kwargs={ 'course_id': unicode(self.course_key), 'student_username': admin.username, 'location': unicode(usage_key), }) response = self.client.get(url) response_content = HTMLParser().unescape(response.content) # We have update the state 4 times: twice to change content, and twice # to set the scores. We'll check that the identifying content from each is # displayed (but not the order), and also the indexes assigned in the output # #1 - #4 self.assertIn('#1', response_content) self.assertIn(json.dumps({'field_a': 'a', 'field_b': 'b'}, sort_keys=True, indent=2), response_content) self.assertIn("Score: 0.0 / 3.0", response_content) self.assertIn(json.dumps({'field_a': 'x', 'field_b': 'y'}, sort_keys=True, indent=2), response_content) self.assertIn("Score: 3.0 / 3.0", response_content) self.assertIn('#4', response_content)
def _build_student_data(cls, user_id, course_key, usage_key_str): """ Generate a list of problem responses for all problem under the ``problem_location`` root. Arguments: user_id (int): The user id for the user generating the report course_key (CourseKey): The ``CourseKey`` for the course whose report is being generated usage_key_str (str): The generated report will include this block and it child blocks. Returns: Tuple[List[Dict], List[str]]: Returns a list of dictionaries containing the student data which will be included in the final csv, and the features/keys to include in that CSV. """ usage_key = UsageKey.from_string(usage_key_str).map_into_course(course_key) user = get_user_model().objects.get(pk=user_id) course_blocks = get_course_blocks(user, usage_key) student_data = [] max_count = settings.FEATURES.get('MAX_PROBLEM_RESPONSES_COUNT') store = modulestore() user_state_client = DjangoXBlockUserStateClient() student_data_keys = set() with store.bulk_operations(course_key): for title, path, block_key in cls._build_problem_list(course_blocks, usage_key): # Chapter and sequential blocks are filtered out since they include state # which isn't useful for this report. if block_key.block_type in ('sequential', 'chapter'): continue block = store.get_item(block_key) generated_report_data = {} # Blocks can implement the generate_report_data method to provide their own # human-readable formatting for user state. if hasattr(block, 'generate_report_data'): try: user_state_iterator = user_state_client.iter_all_for_block(block_key) generated_report_data = { username: state for username, state in block.generate_report_data(user_state_iterator, max_count) } except NotImplementedError: pass responses = list_problem_responses(course_key, block_key, max_count) student_data += responses for response in responses: response['title'] = title # A human-readable location for the current block response['location'] = ' > '.join(path) # A machine-friendly location for the current block response['block_key'] = str(block_key) user_data = generated_report_data.get(response['username'], {}) response.update(user_data) student_data_keys = student_data_keys.union(user_data.keys()) if max_count is not None: max_count -= len(responses) if max_count <= 0: break # Keep the keys in a useful order, starting with username, title and location, # then the columns returned by the xblock report generator in sorted order and # finally end with the more machine friendly block_key and state. student_data_keys_list = ( ['username', 'title', 'location'] + sorted(student_data_keys) + ['block_key', 'state'] ) return student_data, student_data_keys_list
def __init__(self, user, course_id): self._cache = defaultdict(dict) self.course_id = course_id self.user = user self._client = DjangoXBlockUserStateClient(self.user)
class UserStateCache(object): """ Cache for Scope.user_state xblock field data. """ def __init__(self, user, course_id): self._cache = defaultdict(dict) self.course_id = course_id self.user = user self._client = DjangoXBlockUserStateClient(self.user) def cache_fields(self, fields, xblocks, aside_types): # pylint: disable=unused-argument """ Load all fields specified by ``fields`` for the supplied ``xblocks`` and ``aside_types`` into this cache. Arguments: fields (list of str): Field names to cache. xblocks (list of :class:`XBlock`): XBlocks to cache fields for. aside_types (list of str): Aside types to cache fields for. """ block_field_state = self._client.get_many( self.user.username, _all_usage_keys(xblocks, aside_types), ) for usage_key, field_state in block_field_state: self._cache[usage_key] = field_state @contract(kvs_key=DjangoKeyValueStore.Key) def set(self, kvs_key, value): """ Set the specified `kvs_key` to the field value `value`. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete value: The field value to store """ self.set_many({kvs_key: value}) @contract(kvs_key=DjangoKeyValueStore.Key, returns="datetime|None") def last_modified(self, kvs_key): """ Return when the supplied field was changed. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The key representing the cached field Returns: datetime if there was a modified date, or None otherwise """ return self._client.get_mod_date( self.user.username, kvs_key.block_scope_id, fields=[kvs_key.field_name], ).get(kvs_key.field_name) @contract(kv_dict="dict(DjangoKeyValueStore_Key: *)") def set_many(self, kv_dict): """ Set the specified fields to the supplied values. Arguments: kv_dict (dict): A dictionary mapping :class:`~DjangoKeyValueStore.Key` objects to values to set. """ pending_updates = defaultdict(dict) for kvs_key, value in kv_dict.items(): cache_key = self._cache_key_for_kvs_key(kvs_key) pending_updates[cache_key][kvs_key.field_name] = value try: self._client.set_many(self.user.username, pending_updates) except DatabaseError: raise KeyValueMultiSaveError([]) finally: self._cache.update(pending_updates) @contract(kvs_key=DjangoKeyValueStore.Key) def get(self, kvs_key): """ Return the django model object specified by `kvs_key` from the cache. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Returns: A django orm object from the cache """ cache_key = self._cache_key_for_kvs_key(kvs_key) if cache_key not in self._cache: raise KeyError(kvs_key.field_name) return self._cache[cache_key][kvs_key.field_name] @contract(kvs_key=DjangoKeyValueStore.Key) def delete(self, kvs_key): """ Delete the value specified by `kvs_key`. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Raises: KeyError if key isn't found in the cache """ cache_key = self._cache_key_for_kvs_key(kvs_key) if cache_key not in self._cache: raise KeyError(kvs_key.field_name) field_state = self._cache[cache_key] if kvs_key.field_name not in field_state: raise KeyError(kvs_key.field_name) self._client.delete(self.user.username, cache_key, fields=[kvs_key.field_name]) del field_state[kvs_key.field_name] @contract(kvs_key=DjangoKeyValueStore.Key, returns=bool) def has(self, kvs_key): """ Return whether the specified `kvs_key` is set. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Returns: bool """ cache_key = self._cache_key_for_kvs_key(kvs_key) return (cache_key in self._cache and kvs_key.field_name in self._cache[cache_key]) @contract(user_id=int, usage_key=UsageKey, score="number|None", max_score="number|None") def set_score(self, user_id, usage_key, score, max_score): """ UNSUPPORTED METHOD Set the score and max_score for the specified user and xblock usage. """ student_module, created = StudentModule.objects.get_or_create( student_id=user_id, module_state_key=usage_key, course_id=usage_key.course_key, defaults={ 'grade': score, 'max_grade': max_score, }) if not created: student_module.grade = score student_module.max_grade = max_score student_module.save() def __len__(self): return len(self._cache) def _cache_key_for_kvs_key(self, key): """ Return the key used in this DjangoOrmFieldCache for the specified KeyValueStore key. Arguments: key (:class:`~DjangoKeyValueStore.Key`): The key representing the cached field """ return key.block_scope_id
class UserStateCache(object): """ Cache for Scope.user_state xblock field data. """ def __init__(self, user, course_id): self._cache = defaultdict(dict) self.course_id = course_id self.user = user self._client = DjangoXBlockUserStateClient(self.user) def cache_fields(self, fields, xblocks, aside_types): # pylint: disable=unused-argument """ Load all fields specified by ``fields`` for the supplied ``xblocks`` and ``aside_types`` into this cache. Arguments: fields (list of str): Field names to cache. xblocks (list of :class:`XBlock`): XBlocks to cache fields for. aside_types (list of str): Aside types to cache fields for. """ block_field_state = self._client.get_many( self.user.username, _all_usage_keys(xblocks, aside_types), ) for user_state in block_field_state: self._cache[user_state.block_key] = user_state.state @contract(kvs_key=DjangoKeyValueStore.Key) def set(self, kvs_key, value): """ Set the specified `kvs_key` to the field value `value`. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete value: The field value to store """ self.set_many({kvs_key: value}) @contract(kvs_key=DjangoKeyValueStore.Key, returns="datetime|None") def last_modified(self, kvs_key): """ Return when the supplied field was changed. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The key representing the cached field Returns: datetime if there was a modified date, or None otherwise """ try: return self._client.get( self.user.username, kvs_key.block_scope_id, fields=[kvs_key.field_name], ).updated except self._client.DoesNotExist: return None @contract(kv_dict="dict(DjangoKeyValueStore_Key: *)") def set_many(self, kv_dict): """ Set the specified fields to the supplied values. Arguments: kv_dict (dict): A dictionary mapping :class:`~DjangoKeyValueStore.Key` objects to values to set. """ pending_updates = defaultdict(dict) for kvs_key, value in kv_dict.items(): cache_key = self._cache_key_for_kvs_key(kvs_key) pending_updates[cache_key][kvs_key.field_name] = value try: self._client.set_many( self.user.username, pending_updates ) except DatabaseError: log.exception("Saving user state failed for %s", self.user.username) raise KeyValueMultiSaveError([]) finally: self._cache.update(pending_updates) @contract(kvs_key=DjangoKeyValueStore.Key) def get(self, kvs_key): """ Return the django model object specified by `kvs_key` from the cache. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Returns: A django orm object from the cache """ cache_key = self._cache_key_for_kvs_key(kvs_key) if cache_key not in self._cache: raise KeyError(kvs_key.field_name) return self._cache[cache_key][kvs_key.field_name] @contract(kvs_key=DjangoKeyValueStore.Key) def delete(self, kvs_key): """ Delete the value specified by `kvs_key`. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Raises: KeyError if key isn't found in the cache """ cache_key = self._cache_key_for_kvs_key(kvs_key) if cache_key not in self._cache: raise KeyError(kvs_key.field_name) field_state = self._cache[cache_key] if kvs_key.field_name not in field_state: raise KeyError(kvs_key.field_name) self._client.delete(self.user.username, cache_key, fields=[kvs_key.field_name]) del field_state[kvs_key.field_name] @contract(kvs_key=DjangoKeyValueStore.Key, returns=bool) def has(self, kvs_key): """ Return whether the specified `kvs_key` is set. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Returns: bool """ cache_key = self._cache_key_for_kvs_key(kvs_key) return ( cache_key in self._cache and kvs_key.field_name in self._cache[cache_key] ) def __len__(self): return len(self._cache) def _cache_key_for_kvs_key(self, key): """ Return the key used in this DjangoOrmFieldCache for the specified KeyValueStore key. Arguments: key (:class:`~DjangoKeyValueStore.Key`): The key representing the cached field """ return key.block_scope_id
def setUp(self): super(TestDjangoUserStateClient, self).setUp() self.client = DjangoXBlockUserStateClient() self.users = defaultdict(UserFactory.create)
def _build_student_data( cls, user_id, course_key, usage_key_str_list, filter_types=None, ): """ Generate a list of problem responses for all problem under the ``problem_location`` root. Arguments: user_id (int): The user id for the user generating the report course_key (CourseKey): The ``CourseKey`` for the course whose report is being generated usage_key_str_list (List[str]): The generated report will include these blocks and their child blocks. filter_types (List[str]): The report generator will only include data for block types in this list. Returns: Tuple[List[Dict], List[str]]: Returns a list of dictionaries containing the student data which will be included in the final csv, and the features/keys to include in that CSV. """ usage_keys = [ UsageKey.from_string(usage_key_str).map_into_course(course_key) for usage_key_str in usage_key_str_list ] user = get_user_model().objects.get(pk=user_id) student_data = [] max_count = settings.FEATURES.get('MAX_PROBLEM_RESPONSES_COUNT') store = modulestore() user_state_client = DjangoXBlockUserStateClient() student_data_keys = set() with store.bulk_operations(course_key): for usage_key in usage_keys: if max_count is not None and max_count <= 0: break course_blocks = get_course_blocks(user, usage_key) base_path = cls._build_block_base_path( store.get_item(usage_key)) for title, path, block_key in cls._build_problem_list( course_blocks, usage_key): # Chapter and sequential blocks are filtered out since they include state # which isn't useful for this report. if block_key.block_type in ('sequential', 'chapter'): continue if filter_types is not None and block_key.block_type not in filter_types: continue block = store.get_item(block_key) generated_report_data = defaultdict(list) # Blocks can implement the generate_report_data method to provide their own # human-readable formatting for user state. if hasattr(block, 'generate_report_data'): try: user_state_iterator = user_state_client.iter_all_for_block( block_key) for username, state in block.generate_report_data( user_state_iterator, max_count): generated_report_data[username].append(state) except NotImplementedError: pass responses = [] for response in list_problem_responses( course_key, block_key, max_count): response['title'] = title # A human-readable location for the current block response['location'] = ' > '.join(base_path + path) # A machine-friendly location for the current block response['block_key'] = str(block_key) # A block that has a single state per user can contain multiple responses # within the same state. user_states = generated_report_data.get( response['username'], []) if user_states: # For each response in the block, copy over the basic data like the # title, location, block_key and state, and add in the responses for user_state in user_states: user_response = response.copy() user_response.update(user_state) student_data_keys = student_data_keys.union( user_state.keys()) responses.append(user_response) else: responses.append(response) student_data += responses if max_count is not None: max_count -= len(responses) if max_count <= 0: break # Keep the keys in a useful order, starting with username, title and location, # then the columns returned by the xblock report generator in sorted order and # finally end with the more machine friendly block_key and state. student_data_keys_list = (['username', 'title', 'location'] + sorted(student_data_keys) + ['block_key', 'state']) return student_data, student_data_keys_list
def _build_student_data(cls, user_id, course_key, usage_key_str): """ Generate a list of problem responses for all problem under the ``problem_location`` root. Arguments: user_id (int): The user id for the user generating the report course_key (CourseKey): The ``CourseKey`` for the course whose report is being generated usage_key_str (str): The generated report will include this block and it child blocks. Returns: List[Dict]: Returns a list of dictionaries containing the student data which will be included in the final csv. """ usage_key = UsageKey.from_string(usage_key_str).map_into_course( course_key) user = get_user_model().objects.get(pk=user_id) course_blocks = get_course_blocks(user, usage_key) student_data = [] max_count = settings.FEATURES.get('MAX_PROBLEM_RESPONSES_COUNT') store = modulestore() user_state_client = DjangoXBlockUserStateClient() with store.bulk_operations(course_key): for title, path, block_key in cls._build_problem_list( course_blocks, usage_key): # Chapter and sequential blocks are filtered out since they include state # which isn't useful for this report. if block_key.block_type in ('sequential', 'chapter'): continue block = store.get_item(block_key) # Blocks can implement the generate_report_data method to provide their own # human-readable formatting for user state. if hasattr(block, 'generate_report_data'): try: user_state_iterator = user_state_client.iter_all_for_block( block_key) responses = [{ 'username': username, 'state': state } for username, state in block.generate_report_data( user_state_iterator, max_count)] except NotImplementedError: responses = list_problem_responses( course_key, block_key, max_count) else: responses = list_problem_responses(course_key, block_key, max_count) student_data += responses for response in responses: response['title'] = title # A human-readable location for the current block response['location'] = ' > '.join(path) # A machine-friendly location for the current block response['block_key'] = str(block_key) if max_count is not None: max_count -= len(responses) if max_count <= 0: break return student_data
class UserStateCache(object): """ Cache for Scope.user_state xblock field data. """ def __init__(self, user, course_id): self._cache = defaultdict(dict) self.course_id = course_id self.user = user self._client = DjangoXBlockUserStateClient(self.user) def cache_fields(self, fields, xblocks, aside_types): # pylint: disable=unused-argument """ Load all fields specified by ``fields`` for the supplied ``xblocks`` and ``aside_types`` into this cache. Arguments: fields (list of str): Field names to cache. xblocks (list of :class:`XBlock`): XBlocks to cache fields for. aside_types (list of str): Aside types to cache fields for. """ block_field_state = self._client.get_many(self.user.username, _all_usage_keys(xblocks, aside_types)) for usage_key, field_state in block_field_state: self._cache[usage_key] = field_state @contract(kvs_key=DjangoKeyValueStore.Key) def set(self, kvs_key, value): """ Set the specified `kvs_key` to the field value `value`. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete value: The field value to store """ self.set_many({kvs_key: value}) @contract(kvs_key=DjangoKeyValueStore.Key, returns="datetime|None") def last_modified(self, kvs_key): """ Return when the supplied field was changed. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The key representing the cached field Returns: datetime if there was a modified date, or None otherwise """ return self._client.get_mod_date(self.user.username, kvs_key.block_scope_id, fields=[kvs_key.field_name]).get( kvs_key.field_name ) @contract(kv_dict="dict(DjangoKeyValueStore_Key: *)") def set_many(self, kv_dict): """ Set the specified fields to the supplied values. Arguments: kv_dict (dict): A dictionary mapping :class:`~DjangoKeyValueStore.Key` objects to values to set. """ pending_updates = defaultdict(dict) for kvs_key, value in kv_dict.items(): cache_key = self._cache_key_for_kvs_key(kvs_key) pending_updates[cache_key][kvs_key.field_name] = value try: self._client.set_many(self.user.username, pending_updates) except DatabaseError: raise KeyValueMultiSaveError([]) finally: self._cache.update(pending_updates) @contract(kvs_key=DjangoKeyValueStore.Key) def get(self, kvs_key): """ Return the django model object specified by `kvs_key` from the cache. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Returns: A django orm object from the cache """ cache_key = self._cache_key_for_kvs_key(kvs_key) if cache_key not in self._cache: raise KeyError(kvs_key.field_name) return self._cache[cache_key][kvs_key.field_name] @contract(kvs_key=DjangoKeyValueStore.Key) def delete(self, kvs_key): """ Delete the value specified by `kvs_key`. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Raises: KeyError if key isn't found in the cache """ cache_key = self._cache_key_for_kvs_key(kvs_key) if cache_key not in self._cache: raise KeyError(kvs_key.field_name) field_state = self._cache[cache_key] if kvs_key.field_name not in field_state: raise KeyError(kvs_key.field_name) self._client.delete(self.user.username, cache_key, fields=[kvs_key.field_name]) del field_state[kvs_key.field_name] @contract(kvs_key=DjangoKeyValueStore.Key, returns=bool) def has(self, kvs_key): """ Return whether the specified `kvs_key` is set. Arguments: kvs_key (`DjangoKeyValueStore.Key`): The field value to delete Returns: bool """ cache_key = self._cache_key_for_kvs_key(kvs_key) return cache_key in self._cache and kvs_key.field_name in self._cache[cache_key] # @contract(user_id=int, usage_key=UsageKey, score="number|None", max_score="number|None") def set_score(self, user_id, usage_key, score, max_score): """ UNSUPPORTED METHOD Set the score and max_score for the specified user and xblock usage. """ student_module, created = StudentModule.objects.get_or_create( student_id=user_id, module_state_key=usage_key, course_id=usage_key.course_key, defaults={"grade": score, "max_grade": max_score}, ) if not created: student_module.grade = score student_module.max_grade = max_score student_module.save() def __len__(self): return len(self._cache) def _cache_key_for_kvs_key(self, key): """ Return the key used in this DjangoOrmFieldCache for the specified KeyValueStore key. Arguments: key (:class:`~DjangoKeyValueStore.Key`): The key representing the cached field """ return key.block_scope_id