def mergeTranslationMessages(self): """Share `TranslationMessage`s between templates where possible.""" order_check = OrderingCheck( key=methodcaller('sharingKey'), reverse=True) for template_number, template in enumerate(self.potemplates): log.info("Merging template %d/%d." % ( template_number + 1, len(self.potemplates))) deletions = 0 order_check.check(template) potmsgset_ids = self._getPOTMsgSetIds(template) for potmsgset_id in potmsgset_ids: potmsgset = POTMsgSet.get(potmsgset_id) tm_ids = self._partitionTranslationMessageIds(potmsgset) before = sum([len(sublist) for sublist in tm_ids], 0) for ids in tm_ids: for id in ids: message = TranslationMessage.get(id) removeSecurityProxy(message).shareIfPossible() self.tm.endTransaction(intermediate=True) after = potmsgset.getAllTranslationMessages().count() deletions += max(0, before - after) report = "Deleted TranslationMessages: %d." % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def mergeTranslationMessages(self): """Share `TranslationMessage`s between templates where possible.""" order_check = OrderingCheck(key=methodcaller('sharingKey'), reverse=True) for template_number, template in enumerate(self.potemplates): log.info("Merging template %d/%d." % (template_number + 1, len(self.potemplates))) deletions = 0 order_check.check(template) potmsgset_ids = self._getPOTMsgSetIds(template) for potmsgset_id in potmsgset_ids: potmsgset = POTMsgSet.get(potmsgset_id) tm_ids = self._partitionTranslationMessageIds(potmsgset) before = sum([len(sublist) for sublist in tm_ids], 0) for ids in tm_ids: for id in ids: message = TranslationMessage.get(id) removeSecurityProxy(message).shareIfPossible() self.tm.endTransaction(intermediate=True) after = potmsgset.getAllTranslationMessages().count() deletions += max(0, before - after) report = "Deleted TranslationMessages: %d." % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def main(self): tm = TransactionManager(self.txn, False) for packaging in TranslationMerger.findMergeablePackagings(): log.info('Merging %s/%s and %s/%s.' % ( packaging.productseries.product.name, packaging.productseries.name, packaging.sourcepackagename.name, packaging.distroseries.name)) TranslationMerger.mergePackagingTemplates( packaging.productseries, packaging.sourcepackagename, packaging.distroseries, tm) tm.endTransaction(False)
def _scrubPOTMsgSetTranslations(self, potmsgset): """Map out translations for `potmsgset`, and eliminate duplicates. In the transition period for message sharing, there may be duplicate TranslationMessages that may upset assumptions in the code. Clean those up. """ # XXX JeroenVermeulen 2009-06-15 # spec=message-sharing-prevent-duplicates: We're going to have a # unique index again at some point that will prevent this. When # it becomes impossible to test this function, this whole # migration phase can be scrapped. ids_per_language = self._partitionTranslationMessageIds(potmsgset) self.tm.endTransaction(intermediate=True) deletions = 0 for ids in ids_per_language: translations = {} for tm_id in ids: tm = TranslationMessage.get(tm_id) key = self._getPOTMsgSetTranslationMessageKey(tm) if key in translations: language_code = tm.language.code log.info( "Cleaning up identical '%s' message for: \"%s\"" % ( language_code, potmsgset.singular_text)) existing_tm = translations[key] assert tm != existing_tm, ( "Message is duplicate of itself.") assert tm.potmsgset == existing_tm.potmsgset, ( "Different potmsgsets considered identical.") assert tm.potemplate == existing_tm.potemplate, ( "Different potemplates considered identical.") # Transfer any current/imported flags to the existing # message, and delete the duplicate. bequeathe_flags(tm, existing_tm) deletions += 1 else: translations[key] = tm self.tm.endTransaction(intermediate=True) report = "Deleted TranslationMessages: %d" % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def main(self): tm = TransactionManager(self.txn, False) for packaging in TranslationMerger.findMergeablePackagings(): log.info('Merging %s/%s and %s/%s.' % (packaging.productseries.product.name, packaging.productseries.name, packaging.sourcepackagename.name, packaging.distroseries.name)) TranslationMerger.mergePackagingTemplates( packaging.productseries, packaging.sourcepackagename, packaging.distroseries, tm) tm.endTransaction(False)
def _scrubPOTMsgSetTranslations(self, potmsgset): """Map out translations for `potmsgset`, and eliminate duplicates. In the transition period for message sharing, there may be duplicate TranslationMessages that may upset assumptions in the code. Clean those up. """ # XXX JeroenVermeulen 2009-06-15 # spec=message-sharing-prevent-duplicates: We're going to have a # unique index again at some point that will prevent this. When # it becomes impossible to test this function, this whole # migration phase can be scrapped. ids_per_language = self._partitionTranslationMessageIds(potmsgset) self.tm.endTransaction(intermediate=True) deletions = 0 for ids in ids_per_language: translations = {} for tm_id in ids: tm = TranslationMessage.get(tm_id) key = self._getPOTMsgSetTranslationMessageKey(tm) if key in translations: language_code = tm.language.code log.info("Cleaning up identical '%s' message for: \"%s\"" % (language_code, potmsgset.singular_text)) existing_tm = translations[key] assert tm != existing_tm, ( "Message is duplicate of itself.") assert tm.potmsgset == existing_tm.potmsgset, ( "Different potmsgsets considered identical.") assert tm.potemplate == existing_tm.potemplate, ( "Different potemplates considered identical.") # Transfer any current/imported flags to the existing # message, and delete the duplicate. bequeathe_flags(tm, existing_tm) deletions += 1 else: translations[key] = tm self.tm.endTransaction(intermediate=True) report = "Deleted TranslationMessages: %d" % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def mergePOTMsgSets(self): """Merge POTMsgSets for given sequence of sharing templates.""" subordinates, representative_templates = self._mapRepresentatives() num_representatives = len(subordinates) representative_num = 0 for representative, potmsgsets in subordinates.iteritems(): representative_num += 1 log.debug("Message %d/%d: %d subordinate(s)." % ( representative_num, num_representatives, len(potmsgsets))) seen_potmsgsets = set([representative.id]) potmsgset_deletions = 0 tm_deletions = 0 # Merge each subordinate POTMsgSet into its representative. for subordinate in potmsgsets: if subordinate.id in seen_potmsgsets: continue seen_potmsgsets.add(subordinate.id) for message in subordinate.getAllTranslationMessages(): message = removeSecurityProxy(message) clashing_current, clashing_imported, twin = ( self._findClashes( message, representative, message.potemplate)) if clashing_current or clashing_imported: saved = self._saveByDiverging( message, representative, subordinate) else: saved = False if not saved: if twin is None: # This message will have to lose some flags, but # then it can still move to the new potmsgset. sacrifice_flags( message, (clashing_current, clashing_imported)) message.potmsgset = representative else: # This message is identical in contents to one # that was more representative. It'll have to # die, but maybe it can bequeathe some of its # status to the existing message. # Since there are no clashes, there's no need to # check for clashes with other current/imported # messages in the target context. bequeathe_flags( message, twin, (clashing_current, clashing_imported)) tm_deletions += 1 merge_translationtemplateitems( subordinate, representative, representative_templates[representative]) removeSecurityProxy(subordinate).destroySelf() potmsgset_deletions += 1 self.tm.endTransaction(intermediate=True) report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % ( potmsgset_deletions, tm_deletions) if potmsgset_deletions > 0 or tm_deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def main(self): actions = ( self.options.remove_duplicates or self.options.merge_potmsgsets or self.options.merge_translationmessages) if not actions: raise LaunchpadScriptFailure( "Select at least one action: remove duplicates, merge " "POTMsgSets, and/or merge TranslationMessages.") if self.options.product and self.options.distribution: raise LaunchpadScriptFailure( "Merge a product or a distribution, but not both.") if not (self.options.product or self.options.distribution): raise LaunchpadScriptFailure( "Specify a product or distribution to merge.") if self.options.sourcepackage and not self.options.distribution: raise LaunchpadScriptFailure( "Selecting a package only makes sense for distributions.") if self.options.product: product = getUtility(IProductSet).getByName(self.options.product) distribution = None if product is None: raise LaunchpadScriptFailure( "Unknown product: '%s'" % self.options.product) else: product = None # import here to avoid circular import. from lp.registry.interfaces.distribution import IDistributionSet distribution = getUtility(IDistributionSet).getByName( self.options.distribution) if distribution is None: raise LaunchpadScriptFailure( "Unknown distribution: '%s'" % self.options.distribution) if self.options.sourcepackage is None: sourcepackagename = None else: sourcepackagename = getUtility(ISourcePackageNameSet).queryByName( self.options.sourcepackage) if sourcepackagename is None: raise LaunchpadScriptFailure( "Unknown source package name: '%s'" % self.options.sourcepackage) self._setUpUtilities() subset = self.template_set.getSharingSubset( product=product, distribution=distribution, sourcepackagename=sourcepackagename) equivalence_classes = subset.groupEquivalentPOTemplates( self.options.template_names) class_count = len(equivalence_classes) log.info("Merging %d template equivalence classes." % class_count) tm = TransactionManager(self.txn, self.options.dry_run) for number, name in enumerate(sorted(equivalence_classes.iterkeys())): templates = equivalence_classes[name] log.info( "Merging equivalence class '%s': %d template(s) (%d / %d)" % ( name, len(templates), number + 1, class_count)) log.debug("Templates: %s" % str(templates)) merger = TranslationMerger(templates, tm) if self.options.remove_duplicates: log.info("Removing duplicate messages.") merger.removeDuplicateMessages() tm.endTransaction(intermediate=True) if self.options.merge_potmsgsets: log.info("Merging POTMsgSets.") merger.mergePOTMsgSets() tm.endTransaction(intermediate=True) if self.options.merge_translationmessages: log.info("Merging TranslationMessages.") merger.mergeTranslationMessages() tm.endTransaction() log.info("Done.")
def mergePOTMsgSets(self): """Merge POTMsgSets for given sequence of sharing templates.""" subordinates, representative_templates = self._mapRepresentatives() num_representatives = len(subordinates) representative_num = 0 for representative, potmsgsets in subordinates.iteritems(): representative_num += 1 log.debug( "Message %d/%d: %d subordinate(s)." % (representative_num, num_representatives, len(potmsgsets))) seen_potmsgsets = set([representative.id]) potmsgset_deletions = 0 tm_deletions = 0 # Merge each subordinate POTMsgSet into its representative. for subordinate in potmsgsets: if subordinate.id in seen_potmsgsets: continue seen_potmsgsets.add(subordinate.id) for message in subordinate.getAllTranslationMessages(): message = removeSecurityProxy(message) clashing_current, clashing_imported, twin = ( self._findClashes(message, representative, message.potemplate)) if clashing_current or clashing_imported: saved = self._saveByDiverging(message, representative, subordinate) else: saved = False if not saved: if twin is None: # This message will have to lose some flags, but # then it can still move to the new potmsgset. sacrifice_flags( message, (clashing_current, clashing_imported)) message.potmsgset = representative else: # This message is identical in contents to one # that was more representative. It'll have to # die, but maybe it can bequeathe some of its # status to the existing message. # Since there are no clashes, there's no need to # check for clashes with other current/imported # messages in the target context. bequeathe_flags( message, twin, (clashing_current, clashing_imported)) tm_deletions += 1 merge_translationtemplateitems( subordinate, representative, representative_templates[representative]) removeSecurityProxy(subordinate).destroySelf() potmsgset_deletions += 1 self.tm.endTransaction(intermediate=True) report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % ( potmsgset_deletions, tm_deletions) if potmsgset_deletions > 0 or tm_deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def main(self): actions = (self.options.remove_duplicates or self.options.merge_potmsgsets or self.options.merge_translationmessages) if not actions: raise LaunchpadScriptFailure( "Select at least one action: remove duplicates, merge " "POTMsgSets, and/or merge TranslationMessages.") if self.options.product and self.options.distribution: raise LaunchpadScriptFailure( "Merge a product or a distribution, but not both.") if not (self.options.product or self.options.distribution): raise LaunchpadScriptFailure( "Specify a product or distribution to merge.") if self.options.sourcepackage and not self.options.distribution: raise LaunchpadScriptFailure( "Selecting a package only makes sense for distributions.") if self.options.product: product = getUtility(IProductSet).getByName(self.options.product) distribution = None if product is None: raise LaunchpadScriptFailure("Unknown product: '%s'" % self.options.product) else: product = None # import here to avoid circular import. from lp.registry.interfaces.distribution import IDistributionSet distribution = getUtility(IDistributionSet).getByName( self.options.distribution) if distribution is None: raise LaunchpadScriptFailure("Unknown distribution: '%s'" % self.options.distribution) if self.options.sourcepackage is None: sourcepackagename = None else: sourcepackagename = getUtility(ISourcePackageNameSet).queryByName( self.options.sourcepackage) if sourcepackagename is None: raise LaunchpadScriptFailure( "Unknown source package name: '%s'" % self.options.sourcepackage) self._setUpUtilities() subset = self.template_set.getSharingSubset( product=product, distribution=distribution, sourcepackagename=sourcepackagename) template_regex = self.options.template_names if template_regex is not None: template_regex = six.ensure_text(template_regex) equivalence_classes = subset.groupEquivalentPOTemplates(template_regex) class_count = len(equivalence_classes) log.info("Merging %d template equivalence classes." % class_count) tm = TransactionManager(self.txn, self.options.dry_run) for number, name in enumerate(sorted(equivalence_classes.iterkeys())): templates = equivalence_classes[name] log.info( "Merging equivalence class '%s': %d template(s) (%d / %d)" % (name, len(templates), number + 1, class_count)) log.debug("Templates: %s" % str(templates)) merger = TranslationMerger(templates, tm) if self.options.remove_duplicates: log.info("Removing duplicate messages.") merger.removeDuplicateMessages() tm.endTransaction(intermediate=True) if self.options.merge_potmsgsets: log.info("Merging POTMsgSets.") merger.mergePOTMsgSets() tm.endTransaction(intermediate=True) if self.options.merge_translationmessages: log.info("Merging TranslationMessages.") merger.mergeTranslationMessages() tm.endTransaction() log.info("Done.")