Ejemplo n.º 1
0
    def mergeTranslationMessages(self):
        """Share `TranslationMessage`s between templates where possible."""
        order_check = OrderingCheck(
            key=methodcaller('sharingKey'), reverse=True)
        for template_number, template in enumerate(self.potemplates):
            log.info("Merging template %d/%d." % (
                template_number + 1, len(self.potemplates)))
            deletions = 0
            order_check.check(template)
            potmsgset_ids = self._getPOTMsgSetIds(template)
            for potmsgset_id in potmsgset_ids:
                potmsgset = POTMsgSet.get(potmsgset_id)

                tm_ids = self._partitionTranslationMessageIds(potmsgset)
                before = sum([len(sublist) for sublist in tm_ids], 0)

                for ids in tm_ids:
                    for id in ids:
                        message = TranslationMessage.get(id)
                        removeSecurityProxy(message).shareIfPossible()

                self.tm.endTransaction(intermediate=True)

                after = potmsgset.getAllTranslationMessages().count()
                deletions += max(0, before - after)

            report = "Deleted TranslationMessages: %d." % deletions
            if deletions > 0:
                log.info(report)
            else:
                log.log(DEBUG2, report)
Ejemplo n.º 2
0
    def mergeTranslationMessages(self):
        """Share `TranslationMessage`s between templates where possible."""
        order_check = OrderingCheck(key=methodcaller('sharingKey'),
                                    reverse=True)
        for template_number, template in enumerate(self.potemplates):
            log.info("Merging template %d/%d." %
                     (template_number + 1, len(self.potemplates)))
            deletions = 0
            order_check.check(template)
            potmsgset_ids = self._getPOTMsgSetIds(template)
            for potmsgset_id in potmsgset_ids:
                potmsgset = POTMsgSet.get(potmsgset_id)

                tm_ids = self._partitionTranslationMessageIds(potmsgset)
                before = sum([len(sublist) for sublist in tm_ids], 0)

                for ids in tm_ids:
                    for id in ids:
                        message = TranslationMessage.get(id)
                        removeSecurityProxy(message).shareIfPossible()

                self.tm.endTransaction(intermediate=True)

                after = potmsgset.getAllTranslationMessages().count()
                deletions += max(0, before - after)

            report = "Deleted TranslationMessages: %d." % deletions
            if deletions > 0:
                log.info(report)
            else:
                log.log(DEBUG2, report)
Ejemplo n.º 3
0
 def main(self):
     tm = TransactionManager(self.txn, False)
     for packaging in TranslationMerger.findMergeablePackagings():
         log.info('Merging %s/%s and %s/%s.' % (
             packaging.productseries.product.name,
             packaging.productseries.name,
             packaging.sourcepackagename.name,
             packaging.distroseries.name))
         TranslationMerger.mergePackagingTemplates(
             packaging.productseries, packaging.sourcepackagename,
             packaging.distroseries, tm)
         tm.endTransaction(False)
Ejemplo n.º 4
0
    def _scrubPOTMsgSetTranslations(self, potmsgset):
        """Map out translations for `potmsgset`, and eliminate duplicates.

        In the transition period for message sharing, there may be
        duplicate TranslationMessages that may upset assumptions in the
        code.  Clean those up.
        """
        # XXX JeroenVermeulen 2009-06-15
        # spec=message-sharing-prevent-duplicates: We're going to have a
        # unique index again at some point that will prevent this.  When
        # it becomes impossible to test this function, this whole
        # migration phase can be scrapped.
        ids_per_language = self._partitionTranslationMessageIds(potmsgset)

        self.tm.endTransaction(intermediate=True)

        deletions = 0

        for ids in ids_per_language:
            translations = {}

            for tm_id in ids:
                tm = TranslationMessage.get(tm_id)
                key = self._getPOTMsgSetTranslationMessageKey(tm)

                if key in translations:
                    language_code = tm.language.code
                    log.info(
                        "Cleaning up identical '%s' message for: \"%s\"" % (
                            language_code, potmsgset.singular_text))

                    existing_tm = translations[key]
                    assert tm != existing_tm, (
                        "Message is duplicate of itself.")
                    assert tm.potmsgset == existing_tm.potmsgset, (
                        "Different potmsgsets considered identical.")
                    assert tm.potemplate == existing_tm.potemplate, (
                        "Different potemplates considered identical.")

                    # Transfer any current/imported flags to the existing
                    # message, and delete the duplicate.
                    bequeathe_flags(tm, existing_tm)
                    deletions += 1
                else:
                    translations[key] = tm

            self.tm.endTransaction(intermediate=True)

        report = "Deleted TranslationMessages: %d" % deletions
        if deletions > 0:
            log.info(report)
        else:
            log.log(DEBUG2, report)
Ejemplo n.º 5
0
 def main(self):
     tm = TransactionManager(self.txn, False)
     for packaging in TranslationMerger.findMergeablePackagings():
         log.info('Merging %s/%s and %s/%s.' %
                  (packaging.productseries.product.name,
                   packaging.productseries.name,
                   packaging.sourcepackagename.name,
                   packaging.distroseries.name))
         TranslationMerger.mergePackagingTemplates(
             packaging.productseries, packaging.sourcepackagename,
             packaging.distroseries, tm)
         tm.endTransaction(False)
Ejemplo n.º 6
0
    def _scrubPOTMsgSetTranslations(self, potmsgset):
        """Map out translations for `potmsgset`, and eliminate duplicates.

        In the transition period for message sharing, there may be
        duplicate TranslationMessages that may upset assumptions in the
        code.  Clean those up.
        """
        # XXX JeroenVermeulen 2009-06-15
        # spec=message-sharing-prevent-duplicates: We're going to have a
        # unique index again at some point that will prevent this.  When
        # it becomes impossible to test this function, this whole
        # migration phase can be scrapped.
        ids_per_language = self._partitionTranslationMessageIds(potmsgset)

        self.tm.endTransaction(intermediate=True)

        deletions = 0

        for ids in ids_per_language:
            translations = {}

            for tm_id in ids:
                tm = TranslationMessage.get(tm_id)
                key = self._getPOTMsgSetTranslationMessageKey(tm)

                if key in translations:
                    language_code = tm.language.code
                    log.info("Cleaning up identical '%s' message for: \"%s\"" %
                             (language_code, potmsgset.singular_text))

                    existing_tm = translations[key]
                    assert tm != existing_tm, (
                        "Message is duplicate of itself.")
                    assert tm.potmsgset == existing_tm.potmsgset, (
                        "Different potmsgsets considered identical.")
                    assert tm.potemplate == existing_tm.potemplate, (
                        "Different potemplates considered identical.")

                    # Transfer any current/imported flags to the existing
                    # message, and delete the duplicate.
                    bequeathe_flags(tm, existing_tm)
                    deletions += 1
                else:
                    translations[key] = tm

            self.tm.endTransaction(intermediate=True)

        report = "Deleted TranslationMessages: %d" % deletions
        if deletions > 0:
            log.info(report)
        else:
            log.log(DEBUG2, report)
Ejemplo n.º 7
0
    def mergePOTMsgSets(self):
        """Merge POTMsgSets for given sequence of sharing templates."""
        subordinates, representative_templates = self._mapRepresentatives()

        num_representatives = len(subordinates)
        representative_num = 0

        for representative, potmsgsets in subordinates.iteritems():
            representative_num += 1
            log.debug("Message %d/%d: %d subordinate(s)." % (
                representative_num, num_representatives, len(potmsgsets)))

            seen_potmsgsets = set([representative.id])

            potmsgset_deletions = 0
            tm_deletions = 0

            # Merge each subordinate POTMsgSet into its representative.
            for subordinate in potmsgsets:
                if subordinate.id in seen_potmsgsets:
                    continue

                seen_potmsgsets.add(subordinate.id)

                for message in subordinate.getAllTranslationMessages():
                    message = removeSecurityProxy(message)

                    clashing_current, clashing_imported, twin = (
                        self._findClashes(
                            message, representative, message.potemplate))

                    if clashing_current or clashing_imported:
                        saved = self._saveByDiverging(
                            message, representative, subordinate)
                    else:
                        saved = False

                    if not saved:
                        if twin is None:
                            # This message will have to lose some flags, but
                            # then it can still move to the new potmsgset.
                            sacrifice_flags(
                                message,
                                (clashing_current, clashing_imported))
                            message.potmsgset = representative
                        else:
                            # This message is identical in contents to one
                            # that was more representative.  It'll have to
                            # die, but maybe it can bequeathe some of its
                            # status to the existing message.
                            # Since there are no clashes, there's no need to
                            # check for clashes with other current/imported
                            # messages in the target context.
                            bequeathe_flags(
                                message, twin,
                                (clashing_current, clashing_imported))
                            tm_deletions += 1

                merge_translationtemplateitems(
                    subordinate, representative,
                    representative_templates[representative])
                removeSecurityProxy(subordinate).destroySelf()
                potmsgset_deletions += 1

                self.tm.endTransaction(intermediate=True)

            report = "Deleted POTMsgSets: %d.  TranslationMessages: %d." % (
                potmsgset_deletions, tm_deletions)
            if potmsgset_deletions > 0 or tm_deletions > 0:
                log.info(report)
            else:
                log.log(DEBUG2, report)
Ejemplo n.º 8
0
    def main(self):
        actions = (
            self.options.remove_duplicates or
            self.options.merge_potmsgsets or
            self.options.merge_translationmessages)

        if not actions:
            raise LaunchpadScriptFailure(
                "Select at least one action: remove duplicates, merge "
                "POTMsgSets, and/or merge TranslationMessages.")

        if self.options.product and self.options.distribution:
            raise LaunchpadScriptFailure(
                "Merge a product or a distribution, but not both.")

        if not (self.options.product or self.options.distribution):
            raise LaunchpadScriptFailure(
                "Specify a product or distribution to merge.")

        if self.options.sourcepackage and not self.options.distribution:
            raise LaunchpadScriptFailure(
                "Selecting a package only makes sense for distributions.")

        if self.options.product:
            product = getUtility(IProductSet).getByName(self.options.product)
            distribution = None
            if product is None:
                raise LaunchpadScriptFailure(
                    "Unknown product: '%s'" % self.options.product)
        else:
            product = None
            # import here to avoid circular import.
            from lp.registry.interfaces.distribution import IDistributionSet
            distribution = getUtility(IDistributionSet).getByName(
                self.options.distribution)
            if distribution is None:
                raise LaunchpadScriptFailure(
                    "Unknown distribution: '%s'" % self.options.distribution)

        if self.options.sourcepackage is None:
            sourcepackagename = None
        else:
            sourcepackagename = getUtility(ISourcePackageNameSet).queryByName(
                self.options.sourcepackage)
            if sourcepackagename is None:
                raise LaunchpadScriptFailure(
                    "Unknown source package name: '%s'" %
                        self.options.sourcepackage)

        self._setUpUtilities()

        subset = self.template_set.getSharingSubset(
                product=product, distribution=distribution,
                sourcepackagename=sourcepackagename)
        equivalence_classes = subset.groupEquivalentPOTemplates(
                                                self.options.template_names)

        class_count = len(equivalence_classes)
        log.info("Merging %d template equivalence classes." % class_count)

        tm = TransactionManager(self.txn, self.options.dry_run)
        for number, name in enumerate(sorted(equivalence_classes.iterkeys())):
            templates = equivalence_classes[name]
            log.info(
                "Merging equivalence class '%s': %d template(s) (%d / %d)" % (
                    name, len(templates), number + 1, class_count))
            log.debug("Templates: %s" % str(templates))
            merger = TranslationMerger(templates, tm)
            if self.options.remove_duplicates:
                log.info("Removing duplicate messages.")
                merger.removeDuplicateMessages()
                tm.endTransaction(intermediate=True)

            if self.options.merge_potmsgsets:
                log.info("Merging POTMsgSets.")
                merger.mergePOTMsgSets()
                tm.endTransaction(intermediate=True)

            if self.options.merge_translationmessages:
                log.info("Merging TranslationMessages.")
                merger.mergeTranslationMessages()

            tm.endTransaction()

        log.info("Done.")
Ejemplo n.º 9
0
    def mergePOTMsgSets(self):
        """Merge POTMsgSets for given sequence of sharing templates."""
        subordinates, representative_templates = self._mapRepresentatives()

        num_representatives = len(subordinates)
        representative_num = 0

        for representative, potmsgsets in subordinates.iteritems():
            representative_num += 1
            log.debug(
                "Message %d/%d: %d subordinate(s)." %
                (representative_num, num_representatives, len(potmsgsets)))

            seen_potmsgsets = set([representative.id])

            potmsgset_deletions = 0
            tm_deletions = 0

            # Merge each subordinate POTMsgSet into its representative.
            for subordinate in potmsgsets:
                if subordinate.id in seen_potmsgsets:
                    continue

                seen_potmsgsets.add(subordinate.id)

                for message in subordinate.getAllTranslationMessages():
                    message = removeSecurityProxy(message)

                    clashing_current, clashing_imported, twin = (
                        self._findClashes(message, representative,
                                          message.potemplate))

                    if clashing_current or clashing_imported:
                        saved = self._saveByDiverging(message, representative,
                                                      subordinate)
                    else:
                        saved = False

                    if not saved:
                        if twin is None:
                            # This message will have to lose some flags, but
                            # then it can still move to the new potmsgset.
                            sacrifice_flags(
                                message, (clashing_current, clashing_imported))
                            message.potmsgset = representative
                        else:
                            # This message is identical in contents to one
                            # that was more representative.  It'll have to
                            # die, but maybe it can bequeathe some of its
                            # status to the existing message.
                            # Since there are no clashes, there's no need to
                            # check for clashes with other current/imported
                            # messages in the target context.
                            bequeathe_flags(
                                message, twin,
                                (clashing_current, clashing_imported))
                            tm_deletions += 1

                merge_translationtemplateitems(
                    subordinate, representative,
                    representative_templates[representative])
                removeSecurityProxy(subordinate).destroySelf()
                potmsgset_deletions += 1

                self.tm.endTransaction(intermediate=True)

            report = "Deleted POTMsgSets: %d.  TranslationMessages: %d." % (
                potmsgset_deletions, tm_deletions)
            if potmsgset_deletions > 0 or tm_deletions > 0:
                log.info(report)
            else:
                log.log(DEBUG2, report)
Ejemplo n.º 10
0
    def main(self):
        actions = (self.options.remove_duplicates
                   or self.options.merge_potmsgsets
                   or self.options.merge_translationmessages)

        if not actions:
            raise LaunchpadScriptFailure(
                "Select at least one action: remove duplicates, merge "
                "POTMsgSets, and/or merge TranslationMessages.")

        if self.options.product and self.options.distribution:
            raise LaunchpadScriptFailure(
                "Merge a product or a distribution, but not both.")

        if not (self.options.product or self.options.distribution):
            raise LaunchpadScriptFailure(
                "Specify a product or distribution to merge.")

        if self.options.sourcepackage and not self.options.distribution:
            raise LaunchpadScriptFailure(
                "Selecting a package only makes sense for distributions.")

        if self.options.product:
            product = getUtility(IProductSet).getByName(self.options.product)
            distribution = None
            if product is None:
                raise LaunchpadScriptFailure("Unknown product: '%s'" %
                                             self.options.product)
        else:
            product = None
            # import here to avoid circular import.
            from lp.registry.interfaces.distribution import IDistributionSet
            distribution = getUtility(IDistributionSet).getByName(
                self.options.distribution)
            if distribution is None:
                raise LaunchpadScriptFailure("Unknown distribution: '%s'" %
                                             self.options.distribution)

        if self.options.sourcepackage is None:
            sourcepackagename = None
        else:
            sourcepackagename = getUtility(ISourcePackageNameSet).queryByName(
                self.options.sourcepackage)
            if sourcepackagename is None:
                raise LaunchpadScriptFailure(
                    "Unknown source package name: '%s'" %
                    self.options.sourcepackage)

        self._setUpUtilities()

        subset = self.template_set.getSharingSubset(
            product=product,
            distribution=distribution,
            sourcepackagename=sourcepackagename)
        template_regex = self.options.template_names
        if template_regex is not None:
            template_regex = six.ensure_text(template_regex)
        equivalence_classes = subset.groupEquivalentPOTemplates(template_regex)

        class_count = len(equivalence_classes)
        log.info("Merging %d template equivalence classes." % class_count)

        tm = TransactionManager(self.txn, self.options.dry_run)
        for number, name in enumerate(sorted(equivalence_classes.iterkeys())):
            templates = equivalence_classes[name]
            log.info(
                "Merging equivalence class '%s': %d template(s) (%d / %d)" %
                (name, len(templates), number + 1, class_count))
            log.debug("Templates: %s" % str(templates))
            merger = TranslationMerger(templates, tm)
            if self.options.remove_duplicates:
                log.info("Removing duplicate messages.")
                merger.removeDuplicateMessages()
                tm.endTransaction(intermediate=True)

            if self.options.merge_potmsgsets:
                log.info("Merging POTMsgSets.")
                merger.mergePOTMsgSets()
                tm.endTransaction(intermediate=True)

            if self.options.merge_translationmessages:
                log.info("Merging TranslationMessages.")
                merger.mergeTranslationMessages()

            tm.endTransaction()

        log.info("Done.")