def mergeTranslationMessages(self):
"""Share `TranslationMessage`s between templates where possible."""
order_check = OrderingCheck(
key=methodcaller('sharingKey'), reverse=True)
for template_number, template in enumerate(self.potemplates):
log.info("Merging template %d/%d." % (
template_number + 1, len(self.potemplates)))
deletions = 0
order_check.check(template)
potmsgset_ids = self._getPOTMsgSetIds(template)
for potmsgset_id in potmsgset_ids:
potmsgset = POTMsgSet.get(potmsgset_id)
tm_ids = self._partitionTranslationMessageIds(potmsgset)
before = sum([len(sublist) for sublist in tm_ids], 0)
for ids in tm_ids:
for id in ids:
message = TranslationMessage.get(id)
removeSecurityProxy(message).shareIfPossible()
self.tm.endTransaction(intermediate=True)
after = potmsgset.getAllTranslationMessages().count()
deletions += max(0, before - after)
report = "Deleted TranslationMessages: %d." % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def mergeTranslationMessages(self):
"""Share `TranslationMessage`s between templates where possible."""
order_check = OrderingCheck(key=methodcaller('sharingKey'),
reverse=True)
for template_number, template in enumerate(self.potemplates):
log.info("Merging template %d/%d." %
(template_number + 1, len(self.potemplates)))
deletions = 0
order_check.check(template)
potmsgset_ids = self._getPOTMsgSetIds(template)
for potmsgset_id in potmsgset_ids:
potmsgset = POTMsgSet.get(potmsgset_id)
tm_ids = self._partitionTranslationMessageIds(potmsgset)
before = sum([len(sublist) for sublist in tm_ids], 0)
for ids in tm_ids:
for id in ids:
message = TranslationMessage.get(id)
removeSecurityProxy(message).shareIfPossible()
self.tm.endTransaction(intermediate=True)
after = potmsgset.getAllTranslationMessages().count()
deletions += max(0, before - after)
report = "Deleted TranslationMessages: %d." % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def main(self):
tm = TransactionManager(self.txn, False)
for packaging in TranslationMerger.findMergeablePackagings():
log.info('Merging %s/%s and %s/%s.' % (
packaging.productseries.product.name,
packaging.productseries.name,
packaging.sourcepackagename.name,
packaging.distroseries.name))
TranslationMerger.mergePackagingTemplates(
packaging.productseries, packaging.sourcepackagename,
packaging.distroseries, tm)
tm.endTransaction(False)
def _scrubPOTMsgSetTranslations(self, potmsgset):
"""Map out translations for `potmsgset`, and eliminate duplicates.
In the transition period for message sharing, there may be
duplicate TranslationMessages that may upset assumptions in the
code. Clean those up.
"""
# XXX JeroenVermeulen 2009-06-15
# spec=message-sharing-prevent-duplicates: We're going to have a
# unique index again at some point that will prevent this. When
# it becomes impossible to test this function, this whole
# migration phase can be scrapped.
ids_per_language = self._partitionTranslationMessageIds(potmsgset)
self.tm.endTransaction(intermediate=True)
deletions = 0
for ids in ids_per_language:
translations = {}
for tm_id in ids:
tm = TranslationMessage.get(tm_id)
key = self._getPOTMsgSetTranslationMessageKey(tm)
if key in translations:
language_code = tm.language.code
log.info(
"Cleaning up identical '%s' message for: \"%s\"" % (
language_code, potmsgset.singular_text))
existing_tm = translations[key]
assert tm != existing_tm, (
"Message is duplicate of itself.")
assert tm.potmsgset == existing_tm.potmsgset, (
"Different potmsgsets considered identical.")
assert tm.potemplate == existing_tm.potemplate, (
"Different potemplates considered identical.")
# Transfer any current/imported flags to the existing
# message, and delete the duplicate.
bequeathe_flags(tm, existing_tm)
deletions += 1
else:
translations[key] = tm
self.tm.endTransaction(intermediate=True)
report = "Deleted TranslationMessages: %d" % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def main(self):
tm = TransactionManager(self.txn, False)
for packaging in TranslationMerger.findMergeablePackagings():
log.info('Merging %s/%s and %s/%s.' %
(packaging.productseries.product.name,
packaging.productseries.name,
packaging.sourcepackagename.name,
packaging.distroseries.name))
TranslationMerger.mergePackagingTemplates(
packaging.productseries, packaging.sourcepackagename,
packaging.distroseries, tm)
tm.endTransaction(False)
def _scrubPOTMsgSetTranslations(self, potmsgset):
"""Map out translations for `potmsgset`, and eliminate duplicates.
In the transition period for message sharing, there may be
duplicate TranslationMessages that may upset assumptions in the
code. Clean those up.
"""
# XXX JeroenVermeulen 2009-06-15
# spec=message-sharing-prevent-duplicates: We're going to have a
# unique index again at some point that will prevent this. When
# it becomes impossible to test this function, this whole
# migration phase can be scrapped.
ids_per_language = self._partitionTranslationMessageIds(potmsgset)
self.tm.endTransaction(intermediate=True)
deletions = 0
for ids in ids_per_language:
translations = {}
for tm_id in ids:
tm = TranslationMessage.get(tm_id)
key = self._getPOTMsgSetTranslationMessageKey(tm)
if key in translations:
language_code = tm.language.code
log.info("Cleaning up identical '%s' message for: \"%s\"" %
(language_code, potmsgset.singular_text))
existing_tm = translations[key]
assert tm != existing_tm, (
"Message is duplicate of itself.")
assert tm.potmsgset == existing_tm.potmsgset, (
"Different potmsgsets considered identical.")
assert tm.potemplate == existing_tm.potemplate, (
"Different potemplates considered identical.")
# Transfer any current/imported flags to the existing
# message, and delete the duplicate.
bequeathe_flags(tm, existing_tm)
deletions += 1
else:
translations[key] = tm
self.tm.endTransaction(intermediate=True)
report = "Deleted TranslationMessages: %d" % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def mergePOTMsgSets(self):
"""Merge POTMsgSets for given sequence of sharing templates."""
subordinates, representative_templates = self._mapRepresentatives()
num_representatives = len(subordinates)
representative_num = 0
for representative, potmsgsets in subordinates.iteritems():
representative_num += 1
log.debug("Message %d/%d: %d subordinate(s)." % (
representative_num, num_representatives, len(potmsgsets)))
seen_potmsgsets = set([representative.id])
potmsgset_deletions = 0
tm_deletions = 0
# Merge each subordinate POTMsgSet into its representative.
for subordinate in potmsgsets:
if subordinate.id in seen_potmsgsets:
continue
seen_potmsgsets.add(subordinate.id)
for message in subordinate.getAllTranslationMessages():
message = removeSecurityProxy(message)
clashing_current, clashing_imported, twin = (
self._findClashes(
message, representative, message.potemplate))
if clashing_current or clashing_imported:
saved = self._saveByDiverging(
message, representative, subordinate)
else:
saved = False
if not saved:
if twin is None:
# This message will have to lose some flags, but
# then it can still move to the new potmsgset.
sacrifice_flags(
message,
(clashing_current, clashing_imported))
message.potmsgset = representative
else:
# This message is identical in contents to one
# that was more representative. It'll have to
# die, but maybe it can bequeathe some of its
# status to the existing message.
# Since there are no clashes, there's no need to
# check for clashes with other current/imported
# messages in the target context.
bequeathe_flags(
message, twin,
(clashing_current, clashing_imported))
tm_deletions += 1
merge_translationtemplateitems(
subordinate, representative,
representative_templates[representative])
removeSecurityProxy(subordinate).destroySelf()
potmsgset_deletions += 1
self.tm.endTransaction(intermediate=True)
report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % (
potmsgset_deletions, tm_deletions)
if potmsgset_deletions > 0 or tm_deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def main(self):
actions = (
self.options.remove_duplicates or
self.options.merge_potmsgsets or
self.options.merge_translationmessages)
if not actions:
raise LaunchpadScriptFailure(
"Select at least one action: remove duplicates, merge "
"POTMsgSets, and/or merge TranslationMessages.")
if self.options.product and self.options.distribution:
raise LaunchpadScriptFailure(
"Merge a product or a distribution, but not both.")
if not (self.options.product or self.options.distribution):
raise LaunchpadScriptFailure(
"Specify a product or distribution to merge.")
if self.options.sourcepackage and not self.options.distribution:
raise LaunchpadScriptFailure(
"Selecting a package only makes sense for distributions.")
if self.options.product:
product = getUtility(IProductSet).getByName(self.options.product)
distribution = None
if product is None:
raise LaunchpadScriptFailure(
"Unknown product: '%s'" % self.options.product)
else:
product = None
# import here to avoid circular import.
from lp.registry.interfaces.distribution import IDistributionSet
distribution = getUtility(IDistributionSet).getByName(
self.options.distribution)
if distribution is None:
raise LaunchpadScriptFailure(
"Unknown distribution: '%s'" % self.options.distribution)
if self.options.sourcepackage is None:
sourcepackagename = None
else:
sourcepackagename = getUtility(ISourcePackageNameSet).queryByName(
self.options.sourcepackage)
if sourcepackagename is None:
raise LaunchpadScriptFailure(
"Unknown source package name: '%s'" %
self.options.sourcepackage)
self._setUpUtilities()
subset = self.template_set.getSharingSubset(
product=product, distribution=distribution,
sourcepackagename=sourcepackagename)
equivalence_classes = subset.groupEquivalentPOTemplates(
self.options.template_names)
class_count = len(equivalence_classes)
log.info("Merging %d template equivalence classes." % class_count)
tm = TransactionManager(self.txn, self.options.dry_run)
for number, name in enumerate(sorted(equivalence_classes.iterkeys())):
templates = equivalence_classes[name]
log.info(
"Merging equivalence class '%s': %d template(s) (%d / %d)" % (
name, len(templates), number + 1, class_count))
log.debug("Templates: %s" % str(templates))
merger = TranslationMerger(templates, tm)
if self.options.remove_duplicates:
log.info("Removing duplicate messages.")
merger.removeDuplicateMessages()
tm.endTransaction(intermediate=True)
if self.options.merge_potmsgsets:
log.info("Merging POTMsgSets.")
merger.mergePOTMsgSets()
tm.endTransaction(intermediate=True)
if self.options.merge_translationmessages:
log.info("Merging TranslationMessages.")
merger.mergeTranslationMessages()
tm.endTransaction()
log.info("Done.")
def mergePOTMsgSets(self):
"""Merge POTMsgSets for given sequence of sharing templates."""
subordinates, representative_templates = self._mapRepresentatives()
num_representatives = len(subordinates)
representative_num = 0
for representative, potmsgsets in subordinates.iteritems():
representative_num += 1
log.debug(
"Message %d/%d: %d subordinate(s)." %
(representative_num, num_representatives, len(potmsgsets)))
seen_potmsgsets = set([representative.id])
potmsgset_deletions = 0
tm_deletions = 0
# Merge each subordinate POTMsgSet into its representative.
for subordinate in potmsgsets:
if subordinate.id in seen_potmsgsets:
continue
seen_potmsgsets.add(subordinate.id)
for message in subordinate.getAllTranslationMessages():
message = removeSecurityProxy(message)
clashing_current, clashing_imported, twin = (
self._findClashes(message, representative,
message.potemplate))
if clashing_current or clashing_imported:
saved = self._saveByDiverging(message, representative,
subordinate)
else:
saved = False
if not saved:
if twin is None:
# This message will have to lose some flags, but
# then it can still move to the new potmsgset.
sacrifice_flags(
message, (clashing_current, clashing_imported))
message.potmsgset = representative
else:
# This message is identical in contents to one
# that was more representative. It'll have to
# die, but maybe it can bequeathe some of its
# status to the existing message.
# Since there are no clashes, there's no need to
# check for clashes with other current/imported
# messages in the target context.
bequeathe_flags(
message, twin,
(clashing_current, clashing_imported))
tm_deletions += 1
merge_translationtemplateitems(
subordinate, representative,
representative_templates[representative])
removeSecurityProxy(subordinate).destroySelf()
potmsgset_deletions += 1
self.tm.endTransaction(intermediate=True)
report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % (
potmsgset_deletions, tm_deletions)
if potmsgset_deletions > 0 or tm_deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def main(self):
actions = (self.options.remove_duplicates
or self.options.merge_potmsgsets
or self.options.merge_translationmessages)
if not actions:
raise LaunchpadScriptFailure(
"Select at least one action: remove duplicates, merge "
"POTMsgSets, and/or merge TranslationMessages.")
if self.options.product and self.options.distribution:
raise LaunchpadScriptFailure(
"Merge a product or a distribution, but not both.")
if not (self.options.product or self.options.distribution):
raise LaunchpadScriptFailure(
"Specify a product or distribution to merge.")
if self.options.sourcepackage and not self.options.distribution:
raise LaunchpadScriptFailure(
"Selecting a package only makes sense for distributions.")
if self.options.product:
product = getUtility(IProductSet).getByName(self.options.product)
distribution = None
if product is None:
raise LaunchpadScriptFailure("Unknown product: '%s'" %
self.options.product)
else:
product = None
# import here to avoid circular import.
from lp.registry.interfaces.distribution import IDistributionSet
distribution = getUtility(IDistributionSet).getByName(
self.options.distribution)
if distribution is None:
raise LaunchpadScriptFailure("Unknown distribution: '%s'" %
self.options.distribution)
if self.options.sourcepackage is None:
sourcepackagename = None
else:
sourcepackagename = getUtility(ISourcePackageNameSet).queryByName(
self.options.sourcepackage)
if sourcepackagename is None:
raise LaunchpadScriptFailure(
"Unknown source package name: '%s'" %
self.options.sourcepackage)
self._setUpUtilities()
subset = self.template_set.getSharingSubset(
product=product,
distribution=distribution,
sourcepackagename=sourcepackagename)
template_regex = self.options.template_names
if template_regex is not None:
template_regex = six.ensure_text(template_regex)
equivalence_classes = subset.groupEquivalentPOTemplates(template_regex)
class_count = len(equivalence_classes)
log.info("Merging %d template equivalence classes." % class_count)
tm = TransactionManager(self.txn, self.options.dry_run)
for number, name in enumerate(sorted(equivalence_classes.iterkeys())):
templates = equivalence_classes[name]
log.info(
"Merging equivalence class '%s': %d template(s) (%d / %d)" %
(name, len(templates), number + 1, class_count))
log.debug("Templates: %s" % str(templates))
merger = TranslationMerger(templates, tm)
if self.options.remove_duplicates:
log.info("Removing duplicate messages.")
merger.removeDuplicateMessages()
tm.endTransaction(intermediate=True)
if self.options.merge_potmsgsets:
log.info("Merging POTMsgSets.")
merger.mergePOTMsgSets()
tm.endTransaction(intermediate=True)
if self.options.merge_translationmessages:
log.info("Merging TranslationMessages.")
merger.mergeTranslationMessages()
tm.endTransaction()
log.info("Done.")
