def export_data_for_user(user_id): """Exports selected models according to model defined export_data functions. Args: user_id: str. The user_id of the user whose data is being exported. Returns: dict. Dictionary containing all user data in the following format: { <MODEL_NAME>_data: <dict of data in format as specified by model export policy> }. """ user_settings = user_services.get_user_settings(user_id) if user_settings is not None and ( user_settings.role == feconf.ROLE_ID_LEARNER): raise NotImplementedError( 'Takeout for profile users is not yet supported.') exported_data = dict() models_to_export = get_models_which_should_be_exported() for model in models_to_export: split_name = re.findall('[A-Z][^A-Z]*', model.__name__)[:-1] # Join the split name with underscores and add _data for final name. final_name = ('_').join([x.lower() for x in split_name]) exported_data[final_name] = model.export_data(user_id) # Separate out images. We store the images that need to be separated here # as a dictionary mapping tuples to strings. The tuple value indicates the # "path" to take to the image in the user's data dictionary, and the string # indicates the filename that the exported image will be saved to. replacement_instructions = [ takeout_domain.TakeoutImageReplacementInstruction( ('user_settings', 'profile_picture_data_url'), 'user_settings_profile_picture.png', 'profile_picture_filename' ) ] takeout_image_files = [] for replacement_instruction in replacement_instructions: dictionary_path = replacement_instruction.dictionary_path replacement_filename = replacement_instruction.export_filename replacement_key = replacement_instruction.new_key # Move pointer to the position indicated by the tuple. pointer = exported_data for key in dictionary_path[:-1]: pointer = pointer[key] # Swap out data with replacement filename. image_key = dictionary_path[-1] image_data = pointer[image_key] if image_data is not None: takeout_image_files.append( takeout_domain.TakeoutImage(image_data, replacement_filename)) pointer[image_key] = replacement_filename # Rename the key. pointer[replacement_key] = pointer.pop(image_key) return takeout_domain.TakeoutData(exported_data, takeout_image_files)
def export_data_for_user(user_id: str) -> takeout_domain.TakeoutData: """Exports selected models according to model defined export_data functions. Args: user_id: str. The user_id of the user whose data is being exported. Returns: dict. Dictionary containing all user data in the following format: { <MODEL_NAME>_data: <dict of data in format as specified by model export policy> }. Raises: NotImplementedError. Takeout for profile users is not implemented. """ user_settings = user_services.get_user_settings(user_id, strict=False) if user_settings is not None and ( feconf.ROLE_ID_MOBILE_LEARNER in user_settings.roles): raise NotImplementedError( 'Takeout for profile users is not yet supported.') exported_data = {} models_to_export = get_models_which_should_be_exported() for model in models_to_export: split_name = re.findall('[A-Z][^A-Z]*', model.__name__)[:-1] # Join the split name with underscores and add _data for final name. exported_model_data = model.export_data(user_id) exported_model_data_json_string = json.dumps(exported_model_data) user_id_match_object = re.search( feconf.USER_ID_REGEX, exported_model_data_json_string) if user_id_match_object: logging.error( '[TAKEOUT] User ID (%s) found in the JSON generated ' 'for %s and user with ID %s' % ( user_id_match_object.group(0), model.__name__, user_id ) ) final_name = ('_').join([x.lower() for x in split_name]) exported_data[final_name] = exported_model_data # Separate out images. We store the images that need to be separated here # as a dictionary mapping tuples to strings. The tuple value indicates the # "path" to take to the image in the user's data dictionary, and the string # indicates the filename that the exported image will be saved to. replacement_instructions = [ takeout_domain.TakeoutImageReplacementInstruction( ('user_settings', 'profile_picture_data_url'), 'user_settings_profile_picture.png', 'profile_picture_filename' ) ] takeout_image_files: List[takeout_domain.TakeoutImage] = [] for replacement_instruction in replacement_instructions: dictionary_path = replacement_instruction.dictionary_path replacement_filename = replacement_instruction.export_filename replacement_key = replacement_instruction.new_key # Move pointer to the position indicated by the tuple. pointer = exported_data for key in dictionary_path[:-1]: pointer = pointer[key] # Swap out data with replacement filename. image_key = dictionary_path[-1] image_data = pointer[image_key] if image_data is not None: # Ruling out the possibility of Any for mypy type checking. assert isinstance(image_data, str) takeout_image_files.append( takeout_domain.TakeoutImage(image_data, replacement_filename)) pointer[image_key] = replacement_filename # Rename the key. pointer[replacement_key] = pointer.pop(image_key) return takeout_domain.TakeoutData(exported_data, takeout_image_files)
def test_export_data_nontrivial(self): """Nontrivial test of export_data functionality.""" self.set_up_non_trivial() # We set up the feedback_thread_model here so that we can easily # access it when computing the expected data later. feedback_thread_model = feedback_models.GeneralFeedbackThreadModel( entity_type=self.THREAD_ENTITY_TYPE, entity_id=self.THREAD_ENTITY_ID, original_author_id=self.USER_ID_1, status=self.THREAD_STATUS, subject=self.THREAD_SUBJECT, has_suggestion=self.THREAD_HAS_SUGGESTION, summary=self.THREAD_SUMMARY, message_count=self.THREAD_MESSAGE_COUNT) feedback_thread_model.put() expected_stats_data = { 'impact_score': self.USER_1_IMPACT_SCORE, 'total_plays': self.USER_1_TOTAL_PLAYS, 'average_ratings': self.USER_1_AVERAGE_RATINGS, 'num_ratings': self.USER_1_NUM_RATINGS, 'weekly_creator_stats_list': self.USER_1_WEEKLY_CREATOR_STATS_LIST } expected_skill_data = { self.SKILL_ID_1: self.DEGREE_OF_MASTERY, self.SKILL_ID_2: self.DEGREE_OF_MASTERY } expected_contribution_data = { 'created_exploration_ids': [self.EXPLORATION_IDS[0]], 'edited_exploration_ids': [self.EXPLORATION_IDS[0]] } expected_exploration_data = { self.EXPLORATION_IDS[0]: { 'rating': 2, 'rated_on': self.GENERIC_EPOCH, 'draft_change_list': { 'new_content': {} }, 'draft_change_list_last_updated': self.GENERIC_EPOCH, 'draft_change_list_exp_version': 3, 'draft_change_list_id': 1, 'mute_suggestion_notifications': (feconf.DEFAULT_SUGGESTION_NOTIFICATIONS_MUTED_PREFERENCE), 'mute_feedback_notifications': (feconf.DEFAULT_SUGGESTION_NOTIFICATIONS_MUTED_PREFERENCE) } } expected_completed_activities_data = { 'completed_exploration_ids': self.EXPLORATION_IDS, 'completed_collection_ids': self.COLLECTION_IDS } expected_incomplete_activities_data = { 'incomplete_exploration_ids': self.EXPLORATION_IDS, 'incomplete_collection_ids': self.COLLECTION_IDS } expected_last_playthrough_data = { self.EXPLORATION_IDS[0]: { 'exp_version': self.EXP_VERSION, 'state_name': self.STATE_NAME } } expected_learner_playlist_data = { 'playlist_exploration_ids': self.EXPLORATION_IDS, 'playlist_collection_ids': self.COLLECTION_IDS } expected_collection_progress_data = { self.COLLECTION_IDS[0]: self.EXPLORATION_IDS } expected_story_progress_data = { self.STORY_ID_1: self.COMPLETED_NODE_IDS_1 } thread_id = feedback_services.create_thread(self.THREAD_ENTITY_TYPE, self.THREAD_ENTITY_ID, self.USER_ID_1, self.THREAD_SUBJECT, self.MESSAGE_TEXT) feedback_services.create_message(thread_id, self.USER_ID_1, self.THREAD_STATUS, self.THREAD_SUBJECT, self.MESSAGE_TEXT) expected_general_feedback_thread_data = { feedback_thread_model.id: { 'entity_type': self.THREAD_ENTITY_TYPE, 'entity_id': self.THREAD_ENTITY_ID, 'status': self.THREAD_STATUS, 'subject': self.THREAD_SUBJECT, 'has_suggestion': self.THREAD_HAS_SUGGESTION, 'summary': self.THREAD_SUMMARY, 'message_count': self.THREAD_MESSAGE_COUNT, 'last_updated_msec': utils.get_time_in_millisecs(feedback_thread_model.last_updated) }, thread_id: { 'entity_type': self.THREAD_ENTITY_TYPE, 'entity_id': self.THREAD_ENTITY_ID, 'status': self.THREAD_STATUS, 'subject': self.THREAD_SUBJECT, 'has_suggestion': False, 'summary': None, 'message_count': 2, 'last_updated_msec': utils.get_time_in_millisecs( feedback_models.GeneralFeedbackThreadModel.get_by_id( thread_id).last_updated) } } expected_general_feedback_thread_user_data = { thread_id: self.MESSAGE_IDS_READ_BY_USER } expected_general_feedback_message_data = { thread_id + '.0': { 'thread_id': thread_id, 'message_id': 0, 'updated_status': self.THREAD_STATUS, 'updated_subject': self.THREAD_SUBJECT, 'text': self.MESSAGE_TEXT, 'received_via_email': self.MESSAGE_RECEIEVED_VIA_EMAIL }, thread_id + '.1': { 'thread_id': thread_id, 'message_id': 1, 'updated_status': self.THREAD_STATUS, 'updated_subject': self.THREAD_SUBJECT, 'text': self.MESSAGE_TEXT, 'received_via_email': self.MESSAGE_RECEIEVED_VIA_EMAIL } } expected_collection_rights_data = { 'owned_collection_ids': ([self.COLLECTION_IDS[0]]), 'editable_collection_ids': ([self.COLLECTION_IDS[0]]), 'voiced_collection_ids': ([self.COLLECTION_IDS[0]]), 'viewable_collection_ids': [self.COLLECTION_IDS[0]] } expected_general_suggestion_data = { 'exploration.exp1.thread_1': { 'suggestion_type': (suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT), 'target_type': suggestion_models.TARGET_TYPE_EXPLORATION, 'target_id': self.EXPLORATION_IDS[0], 'target_version_at_submission': 1, 'status': suggestion_models.STATUS_IN_REVIEW, 'change_cmd': self.CHANGE_CMD } } expected_exploration_rights_data = { 'owned_exploration_ids': ([self.EXPLORATION_IDS[0]]), 'editable_exploration_ids': ([self.EXPLORATION_IDS[0]]), 'voiced_exploration_ids': ([self.EXPLORATION_IDS[0]]), 'viewable_exploration_ids': [self.EXPLORATION_IDS[0]] } expected_settings_data = { 'email': self.USER_1_EMAIL, 'role': feconf.ROLE_ID_ADMIN, 'username': self.GENERIC_USERNAME, 'normalized_username': self.GENERIC_USERNAME, 'last_agreed_to_terms': self.GENERIC_EPOCH, 'last_started_state_editor_tutorial': self.GENERIC_EPOCH, 'last_started_state_translation_tutorial': self.GENERIC_EPOCH, 'last_logged_in': self.GENERIC_EPOCH, 'last_edited_an_exploration': self.GENERIC_EPOCH, 'profile_picture_filename': 'user_settings_profile_picture.png', 'default_dashboard': 'learner', 'creator_dashboard_display_pref': 'card', 'user_bio': self.GENERIC_USER_BIO, 'subject_interests': self.GENERIC_SUBJECT_INTERESTS, 'first_contribution_msec': 1, 'preferred_language_codes': self.GENERIC_LANGUAGE_CODES, 'preferred_site_language_code': self.GENERIC_LANGUAGE_CODES[0], 'preferred_audio_language_code': self.GENERIC_LANGUAGE_CODES[0] } expected_reply_to_data = { self.THREAD_ID_1: self.USER_1_REPLY_TO_ID_1, self.THREAD_ID_2: self.USER_1_REPLY_TO_ID_2 } expected_subscriptions_data = { 'creator_usernames': self.CREATOR_USERNAMES, 'collection_ids': self.COLLECTION_IDS, 'activity_ids': self.ACTIVITY_IDS + self.EXPLORATION_IDS, 'general_feedback_thread_ids': self.GENERAL_FEEDBACK_THREAD_IDS + [thread_id], 'last_checked': self.GENERIC_EPOCH } expected_task_entry_data = { 'task_ids_resolved_by_user': [self.GENERIC_MODEL_ID] } expected_topic_data = { 'managed_topic_ids': [self.TOPIC_ID_1, self.TOPIC_ID_2] } expected_voiceover_application_data = { 'application_1_id': { 'target_type': 'exploration', 'target_id': 'exp_id', 'status': 'review', 'language_code': 'en', 'filename': 'application_audio.mp3', 'content': '<p>Some content</p>', 'rejection_message': None }, 'application_2_id': { 'target_type': 'exploration', 'target_id': 'exp_id', 'status': 'review', 'language_code': 'en', 'filename': 'application_audio.mp3', 'content': '<p>Some content</p>', 'rejection_message': None } } expected_community_rights_data = { 'can_review_translation_for_language_codes': ['hi', 'en'], 'can_review_voiceover_for_language_codes': ['hi'], 'can_review_questions': True } expected_contrib_score_data = { self.SCORE_CATEGORY_1: { 'has_email_been_sent': False, 'score': 1.5 }, self.SCORE_CATEGORY_2: { 'has_email_been_sent': False, 'score': 2 } } expected_collection_rights_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_collection_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_skill_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_subtopic_page_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_topic_rights_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_topic_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_story_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_question_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_config_property_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_exploration_rights_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_exploration_sm = { 'exp_1-1': { 'commit_type': 'create', 'commit_cmds': [{ 'category': 'A category', 'cmd': 'create_new', 'title': 'A title' }], 'commit_message': 'New exploration created with title \'A title\'.' }, 'exp_1-2': { 'commit_type': 'edit', 'commit_cmds': [{ 'new_value': 'the objective', 'cmd': 'edit_exploration_property', 'old_value': None, 'property_name': 'objective' }], 'commit_message': 'Test edit' } } expected_platform_parameter_sm = { self.GENERIC_MODEL_ID: { 'commit_type': self.COMMIT_TYPE, 'commit_message': self.COMMIT_MESSAGE, 'commit_cmds': self.COMMIT_CMDS } } expected_data = { 'user_stats': expected_stats_data, 'user_settings': expected_settings_data, 'user_subscriptions': expected_subscriptions_data, 'user_skill_mastery': expected_skill_data, 'user_contributions': expected_contribution_data, 'exploration_user_data': expected_exploration_data, 'completed_activities': expected_completed_activities_data, 'incomplete_activities': expected_incomplete_activities_data, 'exp_user_last_playthrough': expected_last_playthrough_data, 'learner_playlist': expected_learner_playlist_data, 'task_entry': expected_task_entry_data, 'topic_rights': expected_topic_data, 'collection_progress': expected_collection_progress_data, 'story_progress': expected_story_progress_data, 'general_feedback_thread': expected_general_feedback_thread_data, 'general_feedback_thread_user': expected_general_feedback_thread_user_data, 'general_feedback_message': expected_general_feedback_message_data, 'collection_rights': expected_collection_rights_data, 'general_suggestion': expected_general_suggestion_data, 'exploration_rights': expected_exploration_rights_data, 'general_feedback_email_reply_to_id': expected_reply_to_data, 'general_voiceover_application': expected_voiceover_application_data, 'user_contribution_scoring': expected_contrib_score_data, 'user_community_rights': expected_community_rights_data, 'collection_rights_snapshot_metadata': expected_collection_rights_sm, 'collection_snapshot_metadata': expected_collection_sm, 'skill_snapshot_metadata': expected_skill_sm, 'subtopic_page_snapshot_metadata': expected_subtopic_page_sm, 'topic_rights_snapshot_metadata': expected_topic_rights_sm, 'topic_snapshot_metadata': expected_topic_sm, 'story_snapshot_metadata': expected_story_sm, 'question_snapshot_metadata': expected_question_sm, 'config_property_snapshot_metadata': expected_config_property_sm, 'exploration_rights_snapshot_metadata': expected_exploration_rights_sm, 'exploration_snapshot_metadata': expected_exploration_sm, 'platform_parameter_snapshot_metadata': expected_platform_parameter_sm, } user_takeout_object = takeout_service.export_data_for_user( self.USER_ID_1) observed_data = user_takeout_object.user_data observed_images = user_takeout_object.user_images self.assertEqual(observed_data, expected_data) observed_json = json.dumps(observed_data) expected_json = json.dumps(expected_data) self.assertEqual(json.loads(observed_json), json.loads(expected_json)) expected_images = [ takeout_domain.TakeoutImage(self.GENERIC_IMAGE_URL, 'user_settings_profile_picture.png') ] self.assertEqual(len(expected_images), len(observed_images)) for i, _ in enumerate(expected_images): self.assertEqual(expected_images[i].b64_image_data, observed_images[i].b64_image_data) self.assertEqual(expected_images[i].image_export_path, observed_images[i].image_export_path)
def test_that_domain_object_is_created_correctly(self) -> None: takeout_image_data = takeout_domain.TakeoutImage( 'b64_fake_image_data', '/test/') self.assertEqual(takeout_image_data.b64_image_data, 'b64_fake_image_data') self.assertEqual(takeout_image_data.image_export_path, '/test/')