def export_data_for_user(user_id):
"""Exports selected models according to model defined export_data functions.
Args:
user_id: str. The user_id of the user whose data is being exported.
Returns:
dict. Dictionary containing all user data in the following format:
{
<MODEL_NAME>_data: <dict of data in format as specified by
model export policy>
}.
"""
user_settings = user_services.get_user_settings(user_id)
if user_settings is not None and (
user_settings.role == feconf.ROLE_ID_LEARNER):
raise NotImplementedError(
'Takeout for profile users is not yet supported.')
exported_data = dict()
models_to_export = get_models_which_should_be_exported()
for model in models_to_export:
split_name = re.findall('[A-Z][^A-Z]*', model.__name__)[:-1]
# Join the split name with underscores and add _data for final name.
final_name = ('_').join([x.lower() for x in split_name])
exported_data[final_name] = model.export_data(user_id)
# Separate out images. We store the images that need to be separated here
# as a dictionary mapping tuples to strings. The tuple value indicates the
# "path" to take to the image in the user's data dictionary, and the string
# indicates the filename that the exported image will be saved to.
replacement_instructions = [
takeout_domain.TakeoutImageReplacementInstruction(
('user_settings', 'profile_picture_data_url'),
'user_settings_profile_picture.png',
'profile_picture_filename'
)
]
takeout_image_files = []
for replacement_instruction in replacement_instructions:
dictionary_path = replacement_instruction.dictionary_path
replacement_filename = replacement_instruction.export_filename
replacement_key = replacement_instruction.new_key
# Move pointer to the position indicated by the tuple.
pointer = exported_data
for key in dictionary_path[:-1]:
pointer = pointer[key]
# Swap out data with replacement filename.
image_key = dictionary_path[-1]
image_data = pointer[image_key]
if image_data is not None:
takeout_image_files.append(
takeout_domain.TakeoutImage(image_data, replacement_filename))
pointer[image_key] = replacement_filename
# Rename the key.
pointer[replacement_key] = pointer.pop(image_key)
return takeout_domain.TakeoutData(exported_data, takeout_image_files)
def export_data_for_user(user_id: str) -> takeout_domain.TakeoutData:
"""Exports selected models according to model defined export_data functions.
Args:
user_id: str. The user_id of the user whose data is being exported.
Returns:
dict. Dictionary containing all user data in the following format:
{
<MODEL_NAME>_data: <dict of data in format as specified by
model export policy>
}.
Raises:
NotImplementedError. Takeout for profile users is not implemented.
"""
user_settings = user_services.get_user_settings(user_id, strict=False)
if user_settings is not None and (
feconf.ROLE_ID_MOBILE_LEARNER in user_settings.roles):
raise NotImplementedError(
'Takeout for profile users is not yet supported.')
exported_data = {}
models_to_export = get_models_which_should_be_exported()
for model in models_to_export:
split_name = re.findall('[A-Z][^A-Z]*', model.__name__)[:-1]
# Join the split name with underscores and add _data for final name.
exported_model_data = model.export_data(user_id)
exported_model_data_json_string = json.dumps(exported_model_data)
user_id_match_object = re.search(
feconf.USER_ID_REGEX, exported_model_data_json_string)
if user_id_match_object:
logging.error(
'[TAKEOUT] User ID (%s) found in the JSON generated '
'for %s and user with ID %s' % (
user_id_match_object.group(0), model.__name__, user_id
)
)
final_name = ('_').join([x.lower() for x in split_name])
exported_data[final_name] = exported_model_data
# Separate out images. We store the images that need to be separated here
# as a dictionary mapping tuples to strings. The tuple value indicates the
# "path" to take to the image in the user's data dictionary, and the string
# indicates the filename that the exported image will be saved to.
replacement_instructions = [
takeout_domain.TakeoutImageReplacementInstruction(
('user_settings', 'profile_picture_data_url'),
'user_settings_profile_picture.png',
'profile_picture_filename'
)
]
takeout_image_files: List[takeout_domain.TakeoutImage] = []
for replacement_instruction in replacement_instructions:
dictionary_path = replacement_instruction.dictionary_path
replacement_filename = replacement_instruction.export_filename
replacement_key = replacement_instruction.new_key
# Move pointer to the position indicated by the tuple.
pointer = exported_data
for key in dictionary_path[:-1]:
pointer = pointer[key]
# Swap out data with replacement filename.
image_key = dictionary_path[-1]
image_data = pointer[image_key]
if image_data is not None:
# Ruling out the possibility of Any for mypy type checking.
assert isinstance(image_data, str)
takeout_image_files.append(
takeout_domain.TakeoutImage(image_data, replacement_filename))
pointer[image_key] = replacement_filename
# Rename the key.
pointer[replacement_key] = pointer.pop(image_key)
return takeout_domain.TakeoutData(exported_data, takeout_image_files)
def test_export_data_nontrivial(self):
"""Nontrivial test of export_data functionality."""
self.set_up_non_trivial()
# We set up the feedback_thread_model here so that we can easily
# access it when computing the expected data later.
feedback_thread_model = feedback_models.GeneralFeedbackThreadModel(
entity_type=self.THREAD_ENTITY_TYPE,
entity_id=self.THREAD_ENTITY_ID,
original_author_id=self.USER_ID_1,
status=self.THREAD_STATUS,
subject=self.THREAD_SUBJECT,
has_suggestion=self.THREAD_HAS_SUGGESTION,
summary=self.THREAD_SUMMARY,
message_count=self.THREAD_MESSAGE_COUNT)
feedback_thread_model.put()
expected_stats_data = {
'impact_score': self.USER_1_IMPACT_SCORE,
'total_plays': self.USER_1_TOTAL_PLAYS,
'average_ratings': self.USER_1_AVERAGE_RATINGS,
'num_ratings': self.USER_1_NUM_RATINGS,
'weekly_creator_stats_list': self.USER_1_WEEKLY_CREATOR_STATS_LIST
}
expected_skill_data = {
self.SKILL_ID_1: self.DEGREE_OF_MASTERY,
self.SKILL_ID_2: self.DEGREE_OF_MASTERY
}
expected_contribution_data = {
'created_exploration_ids': [self.EXPLORATION_IDS[0]],
'edited_exploration_ids': [self.EXPLORATION_IDS[0]]
}
expected_exploration_data = {
self.EXPLORATION_IDS[0]: {
'rating':
2,
'rated_on':
self.GENERIC_EPOCH,
'draft_change_list': {
'new_content': {}
},
'draft_change_list_last_updated':
self.GENERIC_EPOCH,
'draft_change_list_exp_version':
3,
'draft_change_list_id':
1,
'mute_suggestion_notifications':
(feconf.DEFAULT_SUGGESTION_NOTIFICATIONS_MUTED_PREFERENCE),
'mute_feedback_notifications':
(feconf.DEFAULT_SUGGESTION_NOTIFICATIONS_MUTED_PREFERENCE)
}
}
expected_completed_activities_data = {
'completed_exploration_ids': self.EXPLORATION_IDS,
'completed_collection_ids': self.COLLECTION_IDS
}
expected_incomplete_activities_data = {
'incomplete_exploration_ids': self.EXPLORATION_IDS,
'incomplete_collection_ids': self.COLLECTION_IDS
}
expected_last_playthrough_data = {
self.EXPLORATION_IDS[0]: {
'exp_version': self.EXP_VERSION,
'state_name': self.STATE_NAME
}
}
expected_learner_playlist_data = {
'playlist_exploration_ids': self.EXPLORATION_IDS,
'playlist_collection_ids': self.COLLECTION_IDS
}
expected_collection_progress_data = {
self.COLLECTION_IDS[0]: self.EXPLORATION_IDS
}
expected_story_progress_data = {
self.STORY_ID_1: self.COMPLETED_NODE_IDS_1
}
thread_id = feedback_services.create_thread(self.THREAD_ENTITY_TYPE,
self.THREAD_ENTITY_ID,
self.USER_ID_1,
self.THREAD_SUBJECT,
self.MESSAGE_TEXT)
feedback_services.create_message(thread_id, self.USER_ID_1,
self.THREAD_STATUS,
self.THREAD_SUBJECT,
self.MESSAGE_TEXT)
expected_general_feedback_thread_data = {
feedback_thread_model.id: {
'entity_type':
self.THREAD_ENTITY_TYPE,
'entity_id':
self.THREAD_ENTITY_ID,
'status':
self.THREAD_STATUS,
'subject':
self.THREAD_SUBJECT,
'has_suggestion':
self.THREAD_HAS_SUGGESTION,
'summary':
self.THREAD_SUMMARY,
'message_count':
self.THREAD_MESSAGE_COUNT,
'last_updated_msec':
utils.get_time_in_millisecs(feedback_thread_model.last_updated)
},
thread_id: {
'entity_type':
self.THREAD_ENTITY_TYPE,
'entity_id':
self.THREAD_ENTITY_ID,
'status':
self.THREAD_STATUS,
'subject':
self.THREAD_SUBJECT,
'has_suggestion':
False,
'summary':
None,
'message_count':
2,
'last_updated_msec':
utils.get_time_in_millisecs(
feedback_models.GeneralFeedbackThreadModel.get_by_id(
thread_id).last_updated)
}
}
expected_general_feedback_thread_user_data = {
thread_id: self.MESSAGE_IDS_READ_BY_USER
}
expected_general_feedback_message_data = {
thread_id + '.0': {
'thread_id': thread_id,
'message_id': 0,
'updated_status': self.THREAD_STATUS,
'updated_subject': self.THREAD_SUBJECT,
'text': self.MESSAGE_TEXT,
'received_via_email': self.MESSAGE_RECEIEVED_VIA_EMAIL
},
thread_id + '.1': {
'thread_id': thread_id,
'message_id': 1,
'updated_status': self.THREAD_STATUS,
'updated_subject': self.THREAD_SUBJECT,
'text': self.MESSAGE_TEXT,
'received_via_email': self.MESSAGE_RECEIEVED_VIA_EMAIL
}
}
expected_collection_rights_data = {
'owned_collection_ids': ([self.COLLECTION_IDS[0]]),
'editable_collection_ids': ([self.COLLECTION_IDS[0]]),
'voiced_collection_ids': ([self.COLLECTION_IDS[0]]),
'viewable_collection_ids': [self.COLLECTION_IDS[0]]
}
expected_general_suggestion_data = {
'exploration.exp1.thread_1': {
'suggestion_type':
(suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT),
'target_type':
suggestion_models.TARGET_TYPE_EXPLORATION,
'target_id':
self.EXPLORATION_IDS[0],
'target_version_at_submission':
1,
'status':
suggestion_models.STATUS_IN_REVIEW,
'change_cmd':
self.CHANGE_CMD
}
}
expected_exploration_rights_data = {
'owned_exploration_ids': ([self.EXPLORATION_IDS[0]]),
'editable_exploration_ids': ([self.EXPLORATION_IDS[0]]),
'voiced_exploration_ids': ([self.EXPLORATION_IDS[0]]),
'viewable_exploration_ids': [self.EXPLORATION_IDS[0]]
}
expected_settings_data = {
'email': self.USER_1_EMAIL,
'role': feconf.ROLE_ID_ADMIN,
'username': self.GENERIC_USERNAME,
'normalized_username': self.GENERIC_USERNAME,
'last_agreed_to_terms': self.GENERIC_EPOCH,
'last_started_state_editor_tutorial': self.GENERIC_EPOCH,
'last_started_state_translation_tutorial': self.GENERIC_EPOCH,
'last_logged_in': self.GENERIC_EPOCH,
'last_edited_an_exploration': self.GENERIC_EPOCH,
'profile_picture_filename': 'user_settings_profile_picture.png',
'default_dashboard': 'learner',
'creator_dashboard_display_pref': 'card',
'user_bio': self.GENERIC_USER_BIO,
'subject_interests': self.GENERIC_SUBJECT_INTERESTS,
'first_contribution_msec': 1,
'preferred_language_codes': self.GENERIC_LANGUAGE_CODES,
'preferred_site_language_code': self.GENERIC_LANGUAGE_CODES[0],
'preferred_audio_language_code': self.GENERIC_LANGUAGE_CODES[0]
}
expected_reply_to_data = {
self.THREAD_ID_1: self.USER_1_REPLY_TO_ID_1,
self.THREAD_ID_2: self.USER_1_REPLY_TO_ID_2
}
expected_subscriptions_data = {
'creator_usernames':
self.CREATOR_USERNAMES,
'collection_ids':
self.COLLECTION_IDS,
'activity_ids':
self.ACTIVITY_IDS + self.EXPLORATION_IDS,
'general_feedback_thread_ids':
self.GENERAL_FEEDBACK_THREAD_IDS + [thread_id],
'last_checked':
self.GENERIC_EPOCH
}
expected_task_entry_data = {
'task_ids_resolved_by_user': [self.GENERIC_MODEL_ID]
}
expected_topic_data = {
'managed_topic_ids': [self.TOPIC_ID_1, self.TOPIC_ID_2]
}
expected_voiceover_application_data = {
'application_1_id': {
'target_type': 'exploration',
'target_id': 'exp_id',
'status': 'review',
'language_code': 'en',
'filename': 'application_audio.mp3',
'content': '<p>Some content</p>',
'rejection_message': None
},
'application_2_id': {
'target_type': 'exploration',
'target_id': 'exp_id',
'status': 'review',
'language_code': 'en',
'filename': 'application_audio.mp3',
'content': '<p>Some content</p>',
'rejection_message': None
}
}
expected_community_rights_data = {
'can_review_translation_for_language_codes': ['hi', 'en'],
'can_review_voiceover_for_language_codes': ['hi'],
'can_review_questions': True
}
expected_contrib_score_data = {
self.SCORE_CATEGORY_1: {
'has_email_been_sent': False,
'score': 1.5
},
self.SCORE_CATEGORY_2: {
'has_email_been_sent': False,
'score': 2
}
}
expected_collection_rights_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_collection_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_skill_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_subtopic_page_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_topic_rights_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_topic_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_story_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_question_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_config_property_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_exploration_rights_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_exploration_sm = {
'exp_1-1': {
'commit_type':
'create',
'commit_cmds': [{
'category': 'A category',
'cmd': 'create_new',
'title': 'A title'
}],
'commit_message':
'New exploration created with title \'A title\'.'
},
'exp_1-2': {
'commit_type':
'edit',
'commit_cmds': [{
'new_value': 'the objective',
'cmd': 'edit_exploration_property',
'old_value': None,
'property_name': 'objective'
}],
'commit_message':
'Test edit'
}
}
expected_platform_parameter_sm = {
self.GENERIC_MODEL_ID: {
'commit_type': self.COMMIT_TYPE,
'commit_message': self.COMMIT_MESSAGE,
'commit_cmds': self.COMMIT_CMDS
}
}
expected_data = {
'user_stats': expected_stats_data,
'user_settings': expected_settings_data,
'user_subscriptions': expected_subscriptions_data,
'user_skill_mastery': expected_skill_data,
'user_contributions': expected_contribution_data,
'exploration_user_data': expected_exploration_data,
'completed_activities': expected_completed_activities_data,
'incomplete_activities': expected_incomplete_activities_data,
'exp_user_last_playthrough': expected_last_playthrough_data,
'learner_playlist': expected_learner_playlist_data,
'task_entry': expected_task_entry_data,
'topic_rights': expected_topic_data,
'collection_progress': expected_collection_progress_data,
'story_progress': expected_story_progress_data,
'general_feedback_thread': expected_general_feedback_thread_data,
'general_feedback_thread_user':
expected_general_feedback_thread_user_data,
'general_feedback_message': expected_general_feedback_message_data,
'collection_rights': expected_collection_rights_data,
'general_suggestion': expected_general_suggestion_data,
'exploration_rights': expected_exploration_rights_data,
'general_feedback_email_reply_to_id': expected_reply_to_data,
'general_voiceover_application':
expected_voiceover_application_data,
'user_contribution_scoring': expected_contrib_score_data,
'user_community_rights': expected_community_rights_data,
'collection_rights_snapshot_metadata':
expected_collection_rights_sm,
'collection_snapshot_metadata': expected_collection_sm,
'skill_snapshot_metadata': expected_skill_sm,
'subtopic_page_snapshot_metadata': expected_subtopic_page_sm,
'topic_rights_snapshot_metadata': expected_topic_rights_sm,
'topic_snapshot_metadata': expected_topic_sm,
'story_snapshot_metadata': expected_story_sm,
'question_snapshot_metadata': expected_question_sm,
'config_property_snapshot_metadata': expected_config_property_sm,
'exploration_rights_snapshot_metadata':
expected_exploration_rights_sm,
'exploration_snapshot_metadata': expected_exploration_sm,
'platform_parameter_snapshot_metadata':
expected_platform_parameter_sm,
}
user_takeout_object = takeout_service.export_data_for_user(
self.USER_ID_1)
observed_data = user_takeout_object.user_data
observed_images = user_takeout_object.user_images
self.assertEqual(observed_data, expected_data)
observed_json = json.dumps(observed_data)
expected_json = json.dumps(expected_data)
self.assertEqual(json.loads(observed_json), json.loads(expected_json))
expected_images = [
takeout_domain.TakeoutImage(self.GENERIC_IMAGE_URL,
'user_settings_profile_picture.png')
]
self.assertEqual(len(expected_images), len(observed_images))
for i, _ in enumerate(expected_images):
self.assertEqual(expected_images[i].b64_image_data,
observed_images[i].b64_image_data)
self.assertEqual(expected_images[i].image_export_path,
observed_images[i].image_export_path)
def test_that_domain_object_is_created_correctly(self) -> None:
takeout_image_data = takeout_domain.TakeoutImage(
'b64_fake_image_data', '/test/')
self.assertEqual(takeout_image_data.b64_image_data,
'b64_fake_image_data')
self.assertEqual(takeout_image_data.image_export_path, '/test/')
