def mergeTranslationMessages(self):
"""Share `TranslationMessage`s between templates where possible."""
order_check = OrderingCheck(key=methodcaller('sharingKey'),
reverse=True)
for template_number, template in enumerate(self.potemplates):
log.info("Merging template %d/%d." %
(template_number + 1, len(self.potemplates)))
deletions = 0
order_check.check(template)
potmsgset_ids = self._getPOTMsgSetIds(template)
for potmsgset_id in potmsgset_ids:
potmsgset = POTMsgSet.get(potmsgset_id)
tm_ids = self._partitionTranslationMessageIds(potmsgset)
before = sum([len(sublist) for sublist in tm_ids], 0)
for ids in tm_ids:
for id in ids:
message = TranslationMessage.get(id)
removeSecurityProxy(message).shareIfPossible()
self.tm.endTransaction(intermediate=True)
after = potmsgset.getAllTranslationMessages().count()
deletions += max(0, before - after)
report = "Deleted TranslationMessages: %d." % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def mergeTranslationMessages(self):
"""Share `TranslationMessage`s between templates where possible."""
order_check = OrderingCheck(
key=methodcaller('sharingKey'), reverse=True)
for template_number, template in enumerate(self.potemplates):
log.info("Merging template %d/%d." % (
template_number + 1, len(self.potemplates)))
deletions = 0
order_check.check(template)
potmsgset_ids = self._getPOTMsgSetIds(template)
for potmsgset_id in potmsgset_ids:
potmsgset = POTMsgSet.get(potmsgset_id)
tm_ids = self._partitionTranslationMessageIds(potmsgset)
before = sum([len(sublist) for sublist in tm_ids], 0)
for ids in tm_ids:
for id in ids:
message = TranslationMessage.get(id)
removeSecurityProxy(message).shareIfPossible()
self.tm.endTransaction(intermediate=True)
after = potmsgset.getAllTranslationMessages().count()
deletions += max(0, before - after)
report = "Deleted TranslationMessages: %d." % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def _scrubPOTMsgSetTranslations(self, potmsgset):
"""Map out translations for `potmsgset`, and eliminate duplicates.
In the transition period for message sharing, there may be
duplicate TranslationMessages that may upset assumptions in the
code. Clean those up.
"""
# XXX JeroenVermeulen 2009-06-15
# spec=message-sharing-prevent-duplicates: We're going to have a
# unique index again at some point that will prevent this. When
# it becomes impossible to test this function, this whole
# migration phase can be scrapped.
ids_per_language = self._partitionTranslationMessageIds(potmsgset)
self.tm.endTransaction(intermediate=True)
deletions = 0
for ids in ids_per_language:
translations = {}
for tm_id in ids:
tm = TranslationMessage.get(tm_id)
key = self._getPOTMsgSetTranslationMessageKey(tm)
if key in translations:
language_code = tm.language.code
log.info(
"Cleaning up identical '%s' message for: \"%s\"" % (
language_code, potmsgset.singular_text))
existing_tm = translations[key]
assert tm != existing_tm, (
"Message is duplicate of itself.")
assert tm.potmsgset == existing_tm.potmsgset, (
"Different potmsgsets considered identical.")
assert tm.potemplate == existing_tm.potemplate, (
"Different potemplates considered identical.")
# Transfer any current/imported flags to the existing
# message, and delete the duplicate.
bequeathe_flags(tm, existing_tm)
deletions += 1
else:
translations[key] = tm
self.tm.endTransaction(intermediate=True)
report = "Deleted TranslationMessages: %d" % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def _scrubPOTMsgSetTranslations(self, potmsgset):
"""Map out translations for `potmsgset`, and eliminate duplicates.
In the transition period for message sharing, there may be
duplicate TranslationMessages that may upset assumptions in the
code. Clean those up.
"""
# XXX JeroenVermeulen 2009-06-15
# spec=message-sharing-prevent-duplicates: We're going to have a
# unique index again at some point that will prevent this. When
# it becomes impossible to test this function, this whole
# migration phase can be scrapped.
ids_per_language = self._partitionTranslationMessageIds(potmsgset)
self.tm.endTransaction(intermediate=True)
deletions = 0
for ids in ids_per_language:
translations = {}
for tm_id in ids:
tm = TranslationMessage.get(tm_id)
key = self._getPOTMsgSetTranslationMessageKey(tm)
if key in translations:
language_code = tm.language.code
log.info("Cleaning up identical '%s' message for: \"%s\"" %
(language_code, potmsgset.singular_text))
existing_tm = translations[key]
assert tm != existing_tm, (
"Message is duplicate of itself.")
assert tm.potmsgset == existing_tm.potmsgset, (
"Different potmsgsets considered identical.")
assert tm.potemplate == existing_tm.potemplate, (
"Different potemplates considered identical.")
# Transfer any current/imported flags to the existing
# message, and delete the duplicate.
bequeathe_flags(tm, existing_tm)
deletions += 1
else:
translations[key] = tm
self.tm.endTransaction(intermediate=True)
report = "Deleted TranslationMessages: %d" % deletions
if deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def execute_slonik(script, sync=None, exit_on_fail=True, auto_preamble=True):
"""Use the slonik command line tool to run a slonik script.
:param script: The script as a string. Preamble should not be included.
:param sync: Number of seconds to wait for sync before failing. 0 to
block indefinitely.
:param exit_on_fail: If True, on failure of the slonik script
SystemExit is raised using the slonik return code.
:param auto_preamble: If True, the generated preamble will be
automatically included.
:returns: True if the script completed successfully. False if
exit_on_fail is False and the script failed for any reason.
"""
# Add the preamble and optional sync to the script.
if auto_preamble:
script = preamble() + script
if sync is not None:
sync_script = dedent("""\
sync (id = @master_node);
wait for event (
origin = @master_node, confirmed = ALL,
wait on = @master_node, timeout = %d);
""" % sync)
script = script + sync_script
# Copy the script to a NamedTemporaryFile rather than just pumping it
# to slonik via stdin. This way it can be examined if slonik appears
# to hang.
script_on_disk = NamedTemporaryFile(prefix="slonik", suffix=".sk")
print >> script_on_disk, script
script_on_disk.flush()
# Run slonik
log.debug("Executing slonik script %s" % script_on_disk.name)
log.log(DEBUG2, 'Running script:\n%s' % script)
returncode = subprocess.call(['slonik', script_on_disk.name])
if returncode != 0:
log.error("slonik script failed")
if exit_on_fail:
raise SystemExit(1)
return returncode == 0
def execute_slonik(script, sync=None, exit_on_fail=True, auto_preamble=True):
"""Use the slonik command line tool to run a slonik script.
:param script: The script as a string. Preamble should not be included.
:param sync: Number of seconds to wait for sync before failing. 0 to
block indefinitely.
:param exit_on_fail: If True, on failure of the slonik script
SystemExit is raised using the slonik return code.
:param auto_preamble: If True, the generated preamble will be
automatically included.
:returns: True if the script completed successfully. False if
exit_on_fail is False and the script failed for any reason.
"""
# Add the preamble and optional sync to the script.
if auto_preamble:
script = preamble() + script
if sync is not None:
sync_script = dedent("""\
sync (id = @master_node);
wait for event (
origin = @master_node, confirmed = ALL,
wait on = @master_node, timeout = %d);
""" % sync)
script = script + sync_script
# Copy the script to a NamedTemporaryFile rather than just pumping it
# to slonik via stdin. This way it can be examined if slonik appears
# to hang.
script_on_disk = NamedTemporaryFile(prefix="slonik", suffix=".sk")
print >> script_on_disk, script
script_on_disk.flush()
# Run slonik
log.debug("Executing slonik script %s" % script_on_disk.name)
log.log(DEBUG2, 'Running script:\n%s' % script)
returncode = subprocess.call(['slonik', script_on_disk.name])
if returncode != 0:
log.error("slonik script failed")
if exit_on_fail:
raise SystemExit(1)
return returncode == 0
def mergePOTMsgSets(self):
"""Merge POTMsgSets for given sequence of sharing templates."""
subordinates, representative_templates = self._mapRepresentatives()
num_representatives = len(subordinates)
representative_num = 0
for representative, potmsgsets in subordinates.iteritems():
representative_num += 1
log.debug("Message %d/%d: %d subordinate(s)." % (
representative_num, num_representatives, len(potmsgsets)))
seen_potmsgsets = set([representative.id])
potmsgset_deletions = 0
tm_deletions = 0
# Merge each subordinate POTMsgSet into its representative.
for subordinate in potmsgsets:
if subordinate.id in seen_potmsgsets:
continue
seen_potmsgsets.add(subordinate.id)
for message in subordinate.getAllTranslationMessages():
message = removeSecurityProxy(message)
clashing_current, clashing_imported, twin = (
self._findClashes(
message, representative, message.potemplate))
if clashing_current or clashing_imported:
saved = self._saveByDiverging(
message, representative, subordinate)
else:
saved = False
if not saved:
if twin is None:
# This message will have to lose some flags, but
# then it can still move to the new potmsgset.
sacrifice_flags(
message,
(clashing_current, clashing_imported))
message.potmsgset = representative
else:
# This message is identical in contents to one
# that was more representative. It'll have to
# die, but maybe it can bequeathe some of its
# status to the existing message.
# Since there are no clashes, there's no need to
# check for clashes with other current/imported
# messages in the target context.
bequeathe_flags(
message, twin,
(clashing_current, clashing_imported))
tm_deletions += 1
merge_translationtemplateitems(
subordinate, representative,
representative_templates[representative])
removeSecurityProxy(subordinate).destroySelf()
potmsgset_deletions += 1
self.tm.endTransaction(intermediate=True)
report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % (
potmsgset_deletions, tm_deletions)
if potmsgset_deletions > 0 or tm_deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
def mergePOTMsgSets(self):
"""Merge POTMsgSets for given sequence of sharing templates."""
subordinates, representative_templates = self._mapRepresentatives()
num_representatives = len(subordinates)
representative_num = 0
for representative, potmsgsets in subordinates.iteritems():
representative_num += 1
log.debug(
"Message %d/%d: %d subordinate(s)." %
(representative_num, num_representatives, len(potmsgsets)))
seen_potmsgsets = set([representative.id])
potmsgset_deletions = 0
tm_deletions = 0
# Merge each subordinate POTMsgSet into its representative.
for subordinate in potmsgsets:
if subordinate.id in seen_potmsgsets:
continue
seen_potmsgsets.add(subordinate.id)
for message in subordinate.getAllTranslationMessages():
message = removeSecurityProxy(message)
clashing_current, clashing_imported, twin = (
self._findClashes(message, representative,
message.potemplate))
if clashing_current or clashing_imported:
saved = self._saveByDiverging(message, representative,
subordinate)
else:
saved = False
if not saved:
if twin is None:
# This message will have to lose some flags, but
# then it can still move to the new potmsgset.
sacrifice_flags(
message, (clashing_current, clashing_imported))
message.potmsgset = representative
else:
# This message is identical in contents to one
# that was more representative. It'll have to
# die, but maybe it can bequeathe some of its
# status to the existing message.
# Since there are no clashes, there's no need to
# check for clashes with other current/imported
# messages in the target context.
bequeathe_flags(
message, twin,
(clashing_current, clashing_imported))
tm_deletions += 1
merge_translationtemplateitems(
subordinate, representative,
representative_templates[representative])
removeSecurityProxy(subordinate).destroySelf()
potmsgset_deletions += 1
self.tm.endTransaction(intermediate=True)
report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % (
potmsgset_deletions, tm_deletions)
if potmsgset_deletions > 0 or tm_deletions > 0:
log.info(report)
else:
log.log(DEBUG2, report)
