def mergeTranslationMessages(self): """Share `TranslationMessage`s between templates where possible.""" order_check = OrderingCheck(key=methodcaller('sharingKey'), reverse=True) for template_number, template in enumerate(self.potemplates): log.info("Merging template %d/%d." % (template_number + 1, len(self.potemplates))) deletions = 0 order_check.check(template) potmsgset_ids = self._getPOTMsgSetIds(template) for potmsgset_id in potmsgset_ids: potmsgset = POTMsgSet.get(potmsgset_id) tm_ids = self._partitionTranslationMessageIds(potmsgset) before = sum([len(sublist) for sublist in tm_ids], 0) for ids in tm_ids: for id in ids: message = TranslationMessage.get(id) removeSecurityProxy(message).shareIfPossible() self.tm.endTransaction(intermediate=True) after = potmsgset.getAllTranslationMessages().count() deletions += max(0, before - after) report = "Deleted TranslationMessages: %d." % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def mergeTranslationMessages(self): """Share `TranslationMessage`s between templates where possible.""" order_check = OrderingCheck( key=methodcaller('sharingKey'), reverse=True) for template_number, template in enumerate(self.potemplates): log.info("Merging template %d/%d." % ( template_number + 1, len(self.potemplates))) deletions = 0 order_check.check(template) potmsgset_ids = self._getPOTMsgSetIds(template) for potmsgset_id in potmsgset_ids: potmsgset = POTMsgSet.get(potmsgset_id) tm_ids = self._partitionTranslationMessageIds(potmsgset) before = sum([len(sublist) for sublist in tm_ids], 0) for ids in tm_ids: for id in ids: message = TranslationMessage.get(id) removeSecurityProxy(message).shareIfPossible() self.tm.endTransaction(intermediate=True) after = potmsgset.getAllTranslationMessages().count() deletions += max(0, before - after) report = "Deleted TranslationMessages: %d." % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def _scrubPOTMsgSetTranslations(self, potmsgset): """Map out translations for `potmsgset`, and eliminate duplicates. In the transition period for message sharing, there may be duplicate TranslationMessages that may upset assumptions in the code. Clean those up. """ # XXX JeroenVermeulen 2009-06-15 # spec=message-sharing-prevent-duplicates: We're going to have a # unique index again at some point that will prevent this. When # it becomes impossible to test this function, this whole # migration phase can be scrapped. ids_per_language = self._partitionTranslationMessageIds(potmsgset) self.tm.endTransaction(intermediate=True) deletions = 0 for ids in ids_per_language: translations = {} for tm_id in ids: tm = TranslationMessage.get(tm_id) key = self._getPOTMsgSetTranslationMessageKey(tm) if key in translations: language_code = tm.language.code log.info( "Cleaning up identical '%s' message for: \"%s\"" % ( language_code, potmsgset.singular_text)) existing_tm = translations[key] assert tm != existing_tm, ( "Message is duplicate of itself.") assert tm.potmsgset == existing_tm.potmsgset, ( "Different potmsgsets considered identical.") assert tm.potemplate == existing_tm.potemplate, ( "Different potemplates considered identical.") # Transfer any current/imported flags to the existing # message, and delete the duplicate. bequeathe_flags(tm, existing_tm) deletions += 1 else: translations[key] = tm self.tm.endTransaction(intermediate=True) report = "Deleted TranslationMessages: %d" % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def _scrubPOTMsgSetTranslations(self, potmsgset): """Map out translations for `potmsgset`, and eliminate duplicates. In the transition period for message sharing, there may be duplicate TranslationMessages that may upset assumptions in the code. Clean those up. """ # XXX JeroenVermeulen 2009-06-15 # spec=message-sharing-prevent-duplicates: We're going to have a # unique index again at some point that will prevent this. When # it becomes impossible to test this function, this whole # migration phase can be scrapped. ids_per_language = self._partitionTranslationMessageIds(potmsgset) self.tm.endTransaction(intermediate=True) deletions = 0 for ids in ids_per_language: translations = {} for tm_id in ids: tm = TranslationMessage.get(tm_id) key = self._getPOTMsgSetTranslationMessageKey(tm) if key in translations: language_code = tm.language.code log.info("Cleaning up identical '%s' message for: \"%s\"" % (language_code, potmsgset.singular_text)) existing_tm = translations[key] assert tm != existing_tm, ( "Message is duplicate of itself.") assert tm.potmsgset == existing_tm.potmsgset, ( "Different potmsgsets considered identical.") assert tm.potemplate == existing_tm.potemplate, ( "Different potemplates considered identical.") # Transfer any current/imported flags to the existing # message, and delete the duplicate. bequeathe_flags(tm, existing_tm) deletions += 1 else: translations[key] = tm self.tm.endTransaction(intermediate=True) report = "Deleted TranslationMessages: %d" % deletions if deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def execute_slonik(script, sync=None, exit_on_fail=True, auto_preamble=True): """Use the slonik command line tool to run a slonik script. :param script: The script as a string. Preamble should not be included. :param sync: Number of seconds to wait for sync before failing. 0 to block indefinitely. :param exit_on_fail: If True, on failure of the slonik script SystemExit is raised using the slonik return code. :param auto_preamble: If True, the generated preamble will be automatically included. :returns: True if the script completed successfully. False if exit_on_fail is False and the script failed for any reason. """ # Add the preamble and optional sync to the script. if auto_preamble: script = preamble() + script if sync is not None: sync_script = dedent("""\ sync (id = @master_node); wait for event ( origin = @master_node, confirmed = ALL, wait on = @master_node, timeout = %d); """ % sync) script = script + sync_script # Copy the script to a NamedTemporaryFile rather than just pumping it # to slonik via stdin. This way it can be examined if slonik appears # to hang. script_on_disk = NamedTemporaryFile(prefix="slonik", suffix=".sk") print >> script_on_disk, script script_on_disk.flush() # Run slonik log.debug("Executing slonik script %s" % script_on_disk.name) log.log(DEBUG2, 'Running script:\n%s' % script) returncode = subprocess.call(['slonik', script_on_disk.name]) if returncode != 0: log.error("slonik script failed") if exit_on_fail: raise SystemExit(1) return returncode == 0
def mergePOTMsgSets(self): """Merge POTMsgSets for given sequence of sharing templates.""" subordinates, representative_templates = self._mapRepresentatives() num_representatives = len(subordinates) representative_num = 0 for representative, potmsgsets in subordinates.iteritems(): representative_num += 1 log.debug("Message %d/%d: %d subordinate(s)." % ( representative_num, num_representatives, len(potmsgsets))) seen_potmsgsets = set([representative.id]) potmsgset_deletions = 0 tm_deletions = 0 # Merge each subordinate POTMsgSet into its representative. for subordinate in potmsgsets: if subordinate.id in seen_potmsgsets: continue seen_potmsgsets.add(subordinate.id) for message in subordinate.getAllTranslationMessages(): message = removeSecurityProxy(message) clashing_current, clashing_imported, twin = ( self._findClashes( message, representative, message.potemplate)) if clashing_current or clashing_imported: saved = self._saveByDiverging( message, representative, subordinate) else: saved = False if not saved: if twin is None: # This message will have to lose some flags, but # then it can still move to the new potmsgset. sacrifice_flags( message, (clashing_current, clashing_imported)) message.potmsgset = representative else: # This message is identical in contents to one # that was more representative. It'll have to # die, but maybe it can bequeathe some of its # status to the existing message. # Since there are no clashes, there's no need to # check for clashes with other current/imported # messages in the target context. bequeathe_flags( message, twin, (clashing_current, clashing_imported)) tm_deletions += 1 merge_translationtemplateitems( subordinate, representative, representative_templates[representative]) removeSecurityProxy(subordinate).destroySelf() potmsgset_deletions += 1 self.tm.endTransaction(intermediate=True) report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % ( potmsgset_deletions, tm_deletions) if potmsgset_deletions > 0 or tm_deletions > 0: log.info(report) else: log.log(DEBUG2, report)
def mergePOTMsgSets(self): """Merge POTMsgSets for given sequence of sharing templates.""" subordinates, representative_templates = self._mapRepresentatives() num_representatives = len(subordinates) representative_num = 0 for representative, potmsgsets in subordinates.iteritems(): representative_num += 1 log.debug( "Message %d/%d: %d subordinate(s)." % (representative_num, num_representatives, len(potmsgsets))) seen_potmsgsets = set([representative.id]) potmsgset_deletions = 0 tm_deletions = 0 # Merge each subordinate POTMsgSet into its representative. for subordinate in potmsgsets: if subordinate.id in seen_potmsgsets: continue seen_potmsgsets.add(subordinate.id) for message in subordinate.getAllTranslationMessages(): message = removeSecurityProxy(message) clashing_current, clashing_imported, twin = ( self._findClashes(message, representative, message.potemplate)) if clashing_current or clashing_imported: saved = self._saveByDiverging(message, representative, subordinate) else: saved = False if not saved: if twin is None: # This message will have to lose some flags, but # then it can still move to the new potmsgset. sacrifice_flags( message, (clashing_current, clashing_imported)) message.potmsgset = representative else: # This message is identical in contents to one # that was more representative. It'll have to # die, but maybe it can bequeathe some of its # status to the existing message. # Since there are no clashes, there's no need to # check for clashes with other current/imported # messages in the target context. bequeathe_flags( message, twin, (clashing_current, clashing_imported)) tm_deletions += 1 merge_translationtemplateitems( subordinate, representative, representative_templates[representative]) removeSecurityProxy(subordinate).destroySelf() potmsgset_deletions += 1 self.tm.endTransaction(intermediate=True) report = "Deleted POTMsgSets: %d. TranslationMessages: %d." % ( potmsgset_deletions, tm_deletions) if potmsgset_deletions > 0 or tm_deletions > 0: log.info(report) else: log.log(DEBUG2, report)