def test_clear_cache(self): config = Configuration.objects.all().first() get_configuration(code=config.code, edition=config.edition) cache_key = '{}_{}_{}'.format(config.code, config.edition, get_language()) self.assertIsNotNone(cache.get(cache_key)) request = MagicMock() request.user.is_superuser = True # Missing messages-framework will raise a type error. with contextlib.suppress(TypeError): delete_caches(request) self.assertIsNone(cache.get(cache_key))
def test_clear_cache(self): config = Configuration.objects.all().first() get_configuration(code=config.code, edition=config.edition) cache_key = '{}_{}_{}'.format( config.code, config.edition, get_language()) self.assertIsNotNone(cache.get(cache_key)) request = MagicMock() request.user.is_superuser = True # Missing messages-framework will raise a type error. with contextlib.suppress(TypeError): delete_caches(request) self.assertIsNone(cache.get(cache_key))
def __init__(self, instance=None, data=empty, **kwargs): """ Keyword Args: config: configuration.QuestionnaireConfiguration Setup the correct config. If no config is passed, use the questionnaires default config. The config is required for some fields that can only be accessed on the actual configuration object. If a model instance is serialized, the 'instance' kwarg is passed. If an elasticsearch result is deserialized, the 'data' kwarg is passed. """ if instance: self.config = instance.configuration_object elif data != empty and data.get('serializer_config'): # Restore object from json data. Make sure the serializer_config # is valid / exists in the db. self.config = get_configuration(code=data['serializer_config'], edition=data['serializer_edition']) else: raise ValueError( _(u"Can't serialize questionnaire without a valid " u"configuration.")) super().__init__(instance=instance, data=data, **kwargs)
def get_choices_from_questiongroups( questionnaire_data: dict, questiongroups: list, configuration_code: str, configuration_edition: str) -> list: """ Return a list of valid choices based on the presence of certain questiongroups within a questionnaire data JSON. Args: questionnaire_data: The questionnaire data dict questiongroups: A list of questiongroup keywords (basically a list of all possible choices) configuration: QuestionnaireConfiguration Returns: A list of possible choices [(keyword, label)] """ choices = [] configuration = get_configuration( code=configuration_code, edition=configuration_edition) for questiongroup in questiongroups: if questiongroup in questionnaire_data: questiongroup_object = configuration.get_questiongroup_by_keyword( questiongroup) choices.append((questiongroup, questiongroup_object.label)) return choices
def get_choices_from_questiongroups(questionnaire_data: dict, questiongroups: list, configuration_code: str, configuration_edition: str) -> list: """ Return a list of valid choices based on the presence of certain questiongroups within a questionnaire data JSON. Args: questionnaire_data: The questionnaire data dict questiongroups: A list of questiongroup keywords (basically a list of all possible choices) configuration: QuestionnaireConfiguration Returns: A list of possible choices [(keyword, label)] """ choices = [] configuration = get_configuration(code=configuration_code, edition=configuration_edition) for questiongroup in questiongroups: if questiongroup in questionnaire_data: questiongroup_object = configuration.get_questiongroup_by_keyword( questiongroup) choices.append((questiongroup, questiongroup_object.label)) return choices
def setUp(self): super().setUp() self.config = get_configuration(code='approaches', edition='2015') self.questionnaire = Questionnaire.objects.get(id=2) self.data = get_questionnaire_data_in_single_language( self.questionnaire.data, 'en') self.parser = Approach2015Parser(config=self.config, n_a='', questionnaire=self.questionnaire, summary_type='full', **self.data)
def handle(self, **options): pre_save.disconnect(prevent_updates_on_published_items, sender=Questionnaire) configuration_code = 'unccd' config = get_configuration(configuration_code, edition='2015') questionnaires = Questionnaire.objects.filter( code__startswith=configuration_code) self.find_int_values(questionnaires, config) pre_save.connect(prevent_updates_on_published_items, sender=Questionnaire)
def setUp(self): super().setUp() self.config = get_configuration(code='approaches', edition='2015') self.questionnaire = Questionnaire.objects.get(id=2) self.data = get_questionnaire_data_in_single_language( self.questionnaire.data, 'en' ) self.parser = Approach2015Parser( config=self.config, n_a='', questionnaire=self.questionnaire, summary_type='full', **self.data )
def handle(self, **options): pre_save.disconnect( prevent_updates_on_published_items, sender=Questionnaire ) configuration_code = 'unccd' config = get_configuration(configuration_code, edition='2015') questionnaires = Questionnaire.objects.filter( code__startswith=configuration_code) self.find_int_values(questionnaires, config) pre_save.connect( prevent_updates_on_published_items, sender=Questionnaire )
def __init__(self, code: str, edition: str, flat: bool = False): self.structure = [] self.error = None try: configuration = get_configuration(code=code, edition=edition) except Configuration.DoesNotExist: self.error = 'No configuration found for this code and edition.' return for section in configuration.sections: for category in section.categories: if flat: self.build_flat_structure(category) else: self.structure.append(self.get_nested_structure(category))
def links_property(self): """ Collect all info about linked questionnaire and structure it according to language. This follows a often used pattern of questionnaire data. Returns: list """ links = [] current_language = get_language() for link in self.links.filter(status=settings.QUESTIONNAIRE_PUBLIC): link_configuration = get_configuration( code=link.configuration.code, edition=link.configuration.edition) name_data = link_configuration.get_questionnaire_name(link.data) try: original_language = link.questionnairetranslation_set.first( ).language except AttributeError: original_language = settings.LANGUAGES[0][0] # 'en' names = {} urls = {} for code, language in settings.LANGUAGES: activate(code) names[code] = name_data.get(code, name_data.get(original_language)) urls[code] = link.get_absolute_url() if code == original_language: names['default'] = names[code] urls['default'] = urls[code] links.append({ 'code': link.code, 'configuration': link_configuration.keyword, 'name': names, 'url': urls, }) activate(current_language) return links
def handle(self, **options): """ Loop over all active configurations, get the mappings and create the elasticsearch-indexes. Args: **options: None """ configurations = Configuration.objects.all() for language in dict(settings.LANGUAGES).keys(): activate(language) for configuration in configurations: questionnaire_configuration = get_configuration( code=configuration.code, edition=configuration.edition) mappings = get_mappings() create_or_update_index( configuration=questionnaire_configuration, mappings=mappings)
def links_property(self): """ Collect all info about linked questionnaire and structure it according to language. This follows a often used pattern of questionnaire data. Returns: list """ links = [] current_language = get_language() for link in self.links.filter(status=settings.QUESTIONNAIRE_PUBLIC): link_configuration = get_configuration( code=link.configuration.code, edition=link.configuration.edition) name_data = link_configuration.get_questionnaire_name(link.data) try: original_language = link.questionnairetranslation_set.first().language except AttributeError: original_language = settings.LANGUAGES[0][0] # 'en' names = {} urls = {} for code, language in settings.LANGUAGES: activate(code) names[code] = name_data.get(code, name_data.get(original_language)) urls[code] = link.get_absolute_url() if code == original_language: names['default'] = names[code] urls['default'] = urls[code] links.append({ 'code': link.code, 'configuration': link_configuration.keyword, 'name': names, 'url': urls, }) activate(current_language) return links
def index(request, configuration, edition): """ Create or update the mapping of an index. Args: ``request`` (django.http.HttpRequest): The request object. ``configuration`` (str): The code of the Questionnaire configuration. ``edition`` (str): The edition of the Questionnaire configuration. Returns: ``HttpResponse``. A rendered Http Response (redirected to the search admin home page). """ if request.user.is_superuser is not True: raise PermissionDenied() questionnaire_configuration = get_configuration( code=configuration, edition=edition ) if questionnaire_configuration.get_configuration_errors() is not None: return HttpResponseBadRequest( questionnaire_configuration.configuration_error) mappings = get_mappings() success, logs, error_msg = create_or_update_index( configuration=questionnaire_configuration, mappings=mappings ) if success is not True: messages.error(request, 'The following error(s) occured: {}'.format( error_msg)) else: messages.success( request, 'Index "{}" was created or updated.'.format( configuration)) return redirect('search:admin')
def handle(self, **options): """ Loop over all active configurations, get the mappings and create the elasticsearch-indexes. Args: **options: None """ configurations = Configuration.objects.all() for language in dict(settings.LANGUAGES).keys(): activate(language) for configuration in configurations: questionnaire_configuration = get_configuration( code=configuration.code, edition=configuration.edition ) mappings = get_mappings() create_or_update_index( configuration=questionnaire_configuration, mappings=mappings )
def create_temp_indices(configuration_list: list): """ For each index, create the index and update it with the questionnaires of the corresponding configuration as found in the database. Make sure to override the settings and use a custom prefix for the indices. Args: ``configuration_list`` (list): A list of tuples containing the configurations for which a ES index shall be created and consisting of - [0]: code of the configuration - [1]: edition of the configuration """ for code, edition in configuration_list: mappings = get_mappings() create_or_update_index(get_configuration(code=code, edition=edition), mappings) put_questionnaire_data(Questionnaire.with_status.public().filter( configuration__code=code))
def create_temp_indices(configuration_list: list): """ For each index, create the index and update it with the questionnaires of the corresponding configuration as found in the database. Make sure to override the settings and use a custom prefix for the indices. Args: ``configuration_list`` (list): A list of tuples containing the configurations for which a ES index shall be created and consisting of - [0]: code of the configuration - [1]: edition of the configuration """ for code, edition in configuration_list: mappings = get_mappings() create_or_update_index( get_configuration(code=code, edition=edition), mappings) put_questionnaire_data( Questionnaire.with_status.public().filter(configuration__code=code) )
def test_does_not_set_cache_if_found(self, mock_cache): mock_cache.get.return_value = 'bar' get_configuration('foo', 'edition_2015') self.assertEqual(mock_cache.set.call_count, 0)
def test_returns_cache_if_found(self, mock_cache): mock_cache.get.return_value = 'bar' ret = get_configuration('foo', 'edition_2015') self.assertEqual(ret, 'bar')
def post(self, request, *args, **kwargs): if not request.data: # No data passed. return Response({'detail': 'No questionnaire data.'}, status=status.HTTP_400_BAD_REQUEST) # Parse the configuration code and edition request_code = kwargs['configuration'] request_edition = kwargs['edition'] # Validate configuration exists for code and edition structure_obj = ConfigurationStructure( code=request_code, edition=request_edition, ) if structure_obj.error: # No configuration found for this code and edition. return Response( { 'detail': 'No configuration found for this code and edition.' }, status=status.HTTP_404_NOT_FOUND) # Fetch a new configuration request_config = get_configuration(code=request_code, edition=request_edition) # Questionnaires can only be created for the latest edition if request_config.has_new_edition: # Newer edition found for this code return Response( { 'detail': 'A newer edition exists for {}. Questionnaires can be ' 'published only for the latest edition.'.format( request_code) }, status=status.HTTP_400_BAD_REQUEST) # Questionnaire data is validated cleaned_data, config_errors = validate_questionnaire_data( request.data, request_config) if config_errors: # Questionnaire data does not fit configuration structure. return Response({'detail': config_errors}, status=status.HTTP_400_BAD_REQUEST) # Validate the questionnaire data using the serializer serializer = self.get_serializer(data={'data': cleaned_data}) if serializer.is_valid(): # Create a new Questionnaire and save new_questionnaire = Questionnaire.create_new( configuration_code=request_code, data=cleaned_data, user=request.user, created=timezone.now(), updated=timezone.now()) new_questionnaire.save() return Response({ 'success': "true", 'code': new_questionnaire.code }, status=status.HTTP_201_CREATED) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def get(self, request, *args, **kwargs): """ Get a Questionnaire by its identifier for API v2. Returns the questiongroup keywords and its values ``configuration``: The code of the configuration (e.g. "technologies"). ``edition``: The edition of the configuration (e.g. "2018"). ``identifier``: The identifier / code of the questionnaire (e.g. technologies_0123). """ # Parse the configuration code, edition and identifier request_code = kwargs['configuration'] request_edition = kwargs['edition'] request_identifier = kwargs['identifier'] # Validate configuration exists for code and edition structure_obj = ConfigurationStructure( code=request_code, edition=request_edition, ) if structure_obj.error: # No configuration found for this code and edition. return Response( { 'detail': 'No configuration found for this configuration code and edition.' }, status=status.HTTP_400_BAD_REQUEST) # Fetch all questionnaire versions for the identifier questionnaire_all = get_list_or_404( Questionnaire.with_status.not_deleted(), code=request_identifier) # questionnaire = Questionnaire.with_status.not_deleted().filter(code=request_identifier).latest('version') # Use the latest version of the questionnaire if more than one version exists questionnaire = sorted(questionnaire_all, key=lambda x: x.version, reverse=True)[0] if questionnaire is None: # No questionnaire exists or is deleted # - this is a redundant check... raise Http404() # Check configuration matches if not questionnaire.configuration_object == get_configuration( code=request_code, edition=request_edition): # Configuration mismatch return Response( { 'detail': 'Questionnaire does not match configuration code and edition.' }, status=status.HTTP_400_BAD_REQUEST) # Get request user's role in this questionnaire request_user_role = '' for user_role, user in questionnaire.get_users(): if user == request.user: request_user_role = user_role break if not request_user_role: # User has no role on this questionnaire return Response({'detail': 'Unauthorized.'}, status=status.HTTP_401_UNAUTHORIZED) # Check the questionnaire status, allow access only when DRAFT/PUBLIC if questionnaire.status not in (settings.QUESTIONNAIRE_DRAFT, settings.QUESTIONNAIRE_PUBLIC): # Questionnaire is not editable return Response({'detail': 'Not editable.'}, status=status.HTTP_405_METHOD_NOT_ALLOWED) # Check if the questionnaire has any locks if not questionnaire.can_edit(user=request.user): # Questionnaire is currently locked for editing blocked_msg = questionnaire.get_blocked_message( user=request.user)[1] return Response({'detail': blocked_msg}, status=status.HTTP_409_CONFLICT) # Handle the EditRequests table entry # - This request could also be to just view the data, nonetheless we record it as an Edit Request # - Check if the same user has an active EditRequest for this Questionnaire # - if Yes, access timestamp and questionnaire version are updated # - if No, Create a new EditRequest if APIEditRequests.with_status.is_active(code=request_identifier, for_user=request.user): # Found an active request from the user, updating APIEditRequests.objects.filter( questionnaire_code=request_identifier, user=request.user, is_edit_complete=False).update( access=timezone.now(), questionnaire_version=questionnaire.version) else: # No active request from the user, creating APIEditRequests.objects.create( questionnaire_code=request_identifier, questionnaire_version=questionnaire.version, user=request.user) # Finally, deserialize the questionnaire data serializer = QuestionnaireInputSerializer(questionnaire) return Response( { 'status': questionnaire.status_property[0], 'data': serializer.data['data'] }, status=status.HTTP_200_OK)
def handle(self, *args, **options): self.only_on_develop() # Pull latest translations from transifex - unless specified otherwise. if not options.get('do_create_only'): print('Pulling latest from transifex') subprocess.call('tx pull --mode=developer{}'.format( ' -f' if options.get('do_force_pull') else ' ' ), shell=True) print('Compiling latest po files') call_command('compilemessages') configuration_helper_file = os.path.join( 'apps', 'configuration', 'configuration_translations.py' ) # Get a list of all used Translations by all configurations. This can # then be used to remove unused translations. used_translation_ids = [] for configuration in Configuration.objects.all(): questionnaire_configuration = get_configuration( code=configuration.code, edition=configuration.edition) used_translation_ids.extend( questionnaire_configuration.get_translation_ids()) # Remove duplicates used_translation_ids = set(used_translation_ids) # It is not enough to check for new 'Translation' rows as new # configuration editions can update only the translation for the new # edition which creates a new entry in the existing 'Translation' data # json. # Instead, first get all existing content of TranslationContent, group # it by configuration and keyword. Then extract all entries in the # 'Translation' table and see if they are already available in the # 'TranslationContent' table. If not, create them later. existing_translation_content = {} for existing in TranslationContent.objects.all(): (existing_translation_content .setdefault(existing.configuration, {}) .setdefault(existing.keyword, []) .append(existing.text)) unused_translation_ids = [] for translation in Translation.objects.all(): if translation.id not in used_translation_ids: unused_translation_ids.append(translation.id) continue for configuration, contents in translation.data.items(): for keyword, translated_items in contents.items(): # Only English texts are expected. If 'en' is not available, # this must raise an exception. translation_string = translated_items['en'] if translation_string in existing_translation_content.get( configuration, {}).get(keyword, []): # String already in TranslationContent, do nothing. continue # Add new strings to TranslationContent TranslationContent( translation=translation, configuration=configuration, keyword=keyword, text=translation_string ).save() if len(translated_items.keys()) != 1: print(u'Warning: More than one translation in the' u'fixtures. Only the English text is used.') if unused_translation_ids: print( 'The following translations are not needed anymore. They will ' 'be deleted. You should also remove these from the fixtures, ' 'along with the keys/values/categories they belong to (if they ' 'still exist). \n%s' % '\n'.join([str(i) for i in sorted(unused_translation_ids)])) TranslationContent.objects.filter( translation__id__in=unused_translation_ids).delete() Translation.objects.filter(id__in=unused_translation_ids).delete() # All translations must be written to the file again. # By using pgettext and contextual markers, one separate # translation per configuration and keyword is ensured. all_translations = TranslationContent.objects.exclude( text='' ).filter( Q(translation__category__isnull=False) | Q(translation__questiongroup__isnull=False) | Q(translation__key__isnull=False) | Q(translation__value__isnull=False) ).order_by( 'translation__id', 'id' ) with open(configuration_helper_file, 'w') as f: line_number = 1 for translation in all_translations: while line_number < translation.translation.id: f.write('\n') line_number += 1 f.write('pgettext("{0} {1}", {2!r})\n'.format( translation.configuration, translation.keyword, translation.text.replace('\r', '').replace('%', '%%') )) line_number += 1 self.call_parent_makemessages(*args, **options) # Remove temporary file. os.unlink(configuration_helper_file) do_upload_to_transifex = input('Do you want to push the new ' 'translations to transifex? (y/n)') if do_upload_to_transifex == 'y': subprocess.call('tx push -s -t', shell=True)
def post(self, request, *args, **kwargs): """ Update a Questionnaire by its identifier for API v2. Returns the identifier of the updated Questionnaire. The POST data must only contain questiongroup keywords and its values ``configuration``: The code of the configuration (e.g. "technologies"). ``edition``: The edition of the configuration (e.g. "2018"). ``identifier``: The identifier / code of the questionnaire (e.g. technologies_0123). """ if not request.data: # No data passed. return Response({'detail': 'No questionnaire data.'}, status=status.HTTP_400_BAD_REQUEST) # Parse the configuration code, edition and idenfiier request_code = kwargs['configuration'] request_edition = kwargs['edition'] request_identifier = kwargs['identifier'] # Validate configuration exists for code and edition structure_obj = ConfigurationStructure( code=request_code, edition=request_edition, ) if structure_obj.error: # No configuration found for this code and edition. return Response( { 'detail': 'No configuration found for this configuration code and edition.' }, status=status.HTTP_400_BAD_REQUEST) # APIEditRequest corresponding to POST, when the user made the GET, must exist if not APIEditRequests.with_status.is_active(code=request_identifier, for_user=request.user): # No matching edit request, edit is rejected return Response( { 'detail': 'Must fetch questionnaire data before submitting edits.' }, status=status.HTTP_405_METHOD_NOT_ALLOWED) # Fetch all questionnaire versions for the identifier questionnaire_all = get_list_or_404( Questionnaire.with_status.not_deleted(), code=request_identifier) # questionnaire = Questionnaire.with_status.not_deleted().filter(code=request_identifier).latest('version') # Use the latest version of the questionnaire if more than one version exists questionnaire = sorted(questionnaire_all, key=lambda x: x.version, reverse=True)[0] if questionnaire is None: # No questionnaire exists or is deleted # - this is a redundant check... raise Http404() # Fetch a the configuration request_config = get_configuration(code=request_code, edition=request_edition) # Questionnaires can only be created for the latest edition if request_config.has_new_edition: # Newer edition found for this code err = str( 'A newer edition exists for {}. Questionnaires can be published only for the latest edition.' .format(request_code)) return Response({'detail': err}, status=status.HTTP_406_NOT_ACCEPTABLE) # Check configuration matches if questionnaire.configuration_object != request_config: # Configuration mismatch return Response( { 'detail': 'Questionnaire does not match configuration code and edition.' }, status=status.HTTP_400_BAD_REQUEST) # Get request user's role in this questionnaire request_user_role = '' for user_role, user in questionnaire.get_users(): if user == request.user: request_user_role = user_role break if not request_user_role: # User has no role on this questionnaire return Response({'detail': 'Unauthorized.'}, status=status.HTTP_401_UNAUTHORIZED) # Check the questionnaire status, allow access only when DRAFT/PUBLIC if questionnaire.status not in (settings.QUESTIONNAIRE_DRAFT, settings.QUESTIONNAIRE_PUBLIC): # Questionnaire is not editable return Response({'detail': 'Not editable.'}, status=status.HTTP_405_METHOD_NOT_ALLOWED) # Get the EditRequest created for the GET request questionnaire_edit_request = get_object_or_404( APIEditRequests.with_status.is_active(code=request_identifier, for_user=request.user)) # An active request found from the user # - Validate the version and timestamps if not questionnaire_edit_request.questionnaire_version == questionnaire.version: # Newer questionnaire version exists, user has to GET again return Response( {'detail': 'Newer Questionnaire version available.'}, status=status.HTTP_405_METHOD_NOT_ALLOWED) if not questionnaire_edit_request.access > questionnaire.updated: # Questionnaire updated, user has to GET again return Response( {'detail': 'Questionnaire updated since edit started.'}, status=status.HTTP_405_METHOD_NOT_ALLOWED) # Validate the questionnaire edit data cleaned_data, config_errors = validate_questionnaire_data( request.data, request_config) if config_errors: # Questionnaire data does not fit configuration structure. return Response({'detail': config_errors}, status=status.HTTP_400_BAD_REQUEST) if not compare_questionnaire_data(cleaned_data, questionnaire.data): # No changes to Questionnaire data return Response({'detail': "No changes detected."}, status=status.HTTP_400_BAD_REQUEST) # Validate the questionnaire edit data using the serializer serializer = self.get_serializer(data={'data': cleaned_data}) if serializer.is_valid(): # Edit Request is set to complete questionnaire_edit_request.close_request(is_edit_complete=True) # If the questionnaire status is public, we create a new draft version if questionnaire.status == settings.QUESTIONNAIRE_PUBLIC: new_questionnaire = Questionnaire.create_new( configuration_code=questionnaire.configuration.code, data=questionnaire.data, user=self.request.user, previous_version=questionnaire) # Also add previous links to new questionnaire. for linked_questionnaire in questionnaire.links.all(): new_questionnaire.add_link(linked_questionnaire) # 'Draft' Questionnaire data is updated new_questionnaire.update_data( serializer.data['data'], timezone.now(), new_questionnaire.configuration.code) return Response( { 'success': "true", 'code': new_questionnaire.code }, status=status.HTTP_200_OK) # If the questionnaire is a draft version, we accept the edits if questionnaire.status == settings.QUESTIONNAIRE_DRAFT: # Questionnaire data is updated questionnaire.update_data(serializer.data['data'], timezone.now(), questionnaire.configuration.code) return Response({ 'success': "true", 'code': questionnaire.code }, status=status.HTTP_200_OK) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def configuration_object(self): return get_configuration(code=self.configuration.code, edition=self.configuration.edition)
def configuration_object(self): return get_configuration( code=self.configuration.code, edition=self.configuration.edition )
def handle(self, **options): languages = dict(settings.LANGUAGES).keys() for configuration in Configuration.objects.all(): for language in languages: activate(language) get_configuration(configuration.code, configuration.edition)
def handle(self, *args, **options): self.only_on_develop() # Pull latest translations from transifex - unless specified otherwise. if not options.get('do_create_only'): print('Pulling latest from transifex') subprocess.call('tx pull --mode=developer{}'.format( ' -f' if options.get('do_force_pull') else ' '), shell=True) print('Compiling latest po files') call_command('compilemessages') configuration_helper_file = os.path.join( 'apps', 'configuration', 'configuration_translations.py') # Get a list of all used Translations by all configurations. This can # then be used to remove unused translations. used_translation_ids = [] for configuration in Configuration.objects.all(): questionnaire_configuration = get_configuration( code=configuration.code, edition=configuration.edition) used_translation_ids.extend( questionnaire_configuration.get_translation_ids()) # Remove duplicates used_translation_ids = set(used_translation_ids) # It is not enough to check for new 'Translation' rows as new # configuration editions can update only the translation for the new # edition which creates a new entry in the existing 'Translation' data # json. # Instead, first get all existing content of TranslationContent, group # it by configuration and keyword. Then extract all entries in the # 'Translation' table and see if they are already available in the # 'TranslationContent' table. If not, create them later. existing_translation_content = {} for existing in TranslationContent.objects.all(): (existing_translation_content.setdefault( existing.configuration, {}).setdefault(existing.keyword, []).append(existing.text)) unused_translation_ids = [] for translation in Translation.objects.all(): if translation.id not in used_translation_ids: unused_translation_ids.append(translation.id) continue for configuration, contents in translation.data.items(): for keyword, translated_items in contents.items(): # Only English texts are expected. If 'en' is not available, # this must raise an exception. translation_string = translated_items['en'] if translation_string in existing_translation_content.get( configuration, {}).get(keyword, []): # String already in TranslationContent, do nothing. continue # Add new strings to TranslationContent TranslationContent(translation=translation, configuration=configuration, keyword=keyword, text=translation_string).save() if len(translated_items.keys()) != 1: print(u'Warning: More than one translation in the' u'fixtures. Only the English text is used.') if unused_translation_ids: print( 'The following translations are not needed anymore. They will ' 'be deleted. You should also remove these from the fixtures, ' 'along with the keys/values/categories they belong to (if they ' 'still exist). \n%s' % '\n'.join([str(i) for i in sorted(unused_translation_ids)])) TranslationContent.objects.filter( translation__id__in=unused_translation_ids).delete() Translation.objects.filter(id__in=unused_translation_ids).delete() # All translations must be written to the file again. # By using pgettext and contextual markers, one separate # translation per configuration and keyword is ensured. all_translations = TranslationContent.objects.exclude(text='').filter( Q(translation__category__isnull=False) | Q(translation__questiongroup__isnull=False) | Q(translation__key__isnull=False) | Q(translation__value__isnull=False)).order_by( 'translation__id', 'id') with open(configuration_helper_file, 'w') as f: line_number = 1 for translation in all_translations: while line_number < translation.translation.id: f.write('\n') line_number += 1 f.write('pgettext("{0} {1}", {2!r})\n'.format( translation.configuration, translation.keyword, translation.text.replace('\r', '').replace('%', '%%'))) line_number += 1 self.call_parent_makemessages(*args, **options) # Remove temporary file. os.unlink(configuration_helper_file) do_upload_to_transifex = input('Do you want to push the new ' 'translations to transifex? (y/n)') if do_upload_to_transifex == 'y': subprocess.call('tx push -s -t', shell=True)