def __setattr__(self, key, value): if key == 'wordTypeID': if self.wordTypeID: wordType = locWordType.WordType.Get(self.wordTypeID) wordTypeName = wordType.typeName if wordType else 'None' raise AuthoringValidationError( "Cannot change wordTypeID: Group '%s' (groupID %s) may contain metadata for wordType '%s'; call ResetWordType first to delete all metadata in this group and try again." % (self.groupName, self.groupID, wordTypeName)) if locWordType.WordType.Get(value) == None: raise AuthoringValidationError( 'WordTypeID (%s) does not exist.' % value) with bsd.BsdTransaction(): for message in locMessage.Message.GetMessagesByGroupID( self.groupID): message.wordTypeID = value bsdWrappers.BaseWrapper.__setattr__(self, key, value) return if key == 'parentID' and value is not None: if not MessageGroup.Get(value): raise AuthoringValidationError( "Cannot set parentID: '%s' is not a valid groupID." % value) if self._IsSubGroup(value): subGroup = MessageGroup.Get(value) raise AuthoringValidationError( "You cannot assign group '%s' as a child of group '%s' because it would create a circular reference." % (self.groupName, subGroup.groupName)) bsdWrappers.BaseWrapper.__setattr__(self, key, value)
def ResetWordType(self): with bsd.BsdTransaction(): for message in locMessage.Message.GetMessagesByGroupID( self.groupID): message.ResetWordType() bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def _CopyAllMetaDataToNewMessage(cls, sourceMessageID, destMessageID, destinationWordTypeID, transactionBundle=None): """ Copy all metadata entries to a newly created message (that may not exist yet) This method is internal for all purposes prereq: sourceMessageID must exist in BSD return: True on success """ cls.CheckAndSetCache() sourceMetaRows = cls._metaDataTable.GetRows(messageID=sourceMessageID, _getDeleted=False) with bsd.BsdTransaction(): for aMetaRow in sourceMetaRows: propertyRow = cls._propertyTable.GetRowByKey( aMetaRow.wordPropertyID) if propertyRow.wordTypeID != destinationWordTypeID: raise AuthoringValidationError( 'Source metadata property typeID doesnt match destination typeID. (%s, %s).' % (propertyRow.wordTypeID, destinationWordTypeID)) cls._TransactionAwareCreate( wordPropertyID=aMetaRow.wordPropertyID, messageID=destMessageID, metaDataValue=aMetaRow.metaDataValue, transactionBundle=transactionBundle) return True
def AddMetaDataEntryByName(self, propertyName, languageID, metaDataValue, transactionBundle=None): """ Add new metadata entry to this message. Will throw an error if the metadata entry already exists. NOTE: This code is bsd transaction friendly. parameters: propertyName - name of the property languageID - id for the language. For example "en-us" metaDataValue - metadata string transactionBundle - cache containing entries of messages to be added within transaction. It is required for validations within transactions. See CreateMessageDataBundle() """ if self.wordTypeID == None: raise AuthoringValidationError( 'Before adding metadata, the wordType needs to be set on this messageID (%s).' % str(self.messageID)) typeRow = locWordType.WordType.Get(self.wordTypeID) if typeRow == None: raise AuthoringValidationError( 'WordTypeID (%s), of this message, does not exist.' % self.wordTypeID) typeName = typeRow.typeName with bsd.BsdTransaction(): locWordMetaData.WordMetaData.TransactionAwareCreateFromPropertyName( typeName, propertyName, languageID, self.messageID, metaDataValue, transactionBundle)
def Create(cls, label, groupID=None, text='', context=None): """ Creates a new message label, and a blank English string by default. parameters: label - label of the message. Must be unique groupID - destination group. Must match the wordType of this message text - english text context - description for this message entry pre-req: this MUST NOT be called within nested Transactions. This not what this method for. exceptions: throws various exceptions on failures returns: row of the message that we just added """ cls._ErrorIfInTransaction( 'Message Create will not run within Transaction. Use TransactionAwareCreate.' ) with bsd.BsdTransaction('Creating new message: %s' % label) as bsdTransaction: cls._TransactionAwareCreate(label, groupID, LOCALE_SHORT_ENGLISH, text, context, wordTypeID=None, transactionBundle=None) resultList = bsdTransaction.GetTransactionResult() return cls.Get(resultList[0][1].messageID)
def ResetWordType(self): """ Change the wordTypeID of this message to None. This operation will remove all metadata belonging to this message. NOTE: This code is bsd transaction friendly. """ if self.wordTypeID is not None: with bsd.BsdTransaction(): self._DeleteMetaData() bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def ResetWordType(self): """ Deletes all existing metadata on all messages in the group, sets their word type to None, and sets the word type of the group to None. """ with bsd.BsdTransaction(): for message in locMessage.Message.GetMessagesByGroupID( self.groupID): message.ResetWordType() bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def AddMetaDataEntry(self, wordPropertyID, metaDataValue, transactionBundle=None): if self.wordTypeID == None: raise AuthoringValidationError( 'Before adding metadata, the wordType needs to be set on this messageID (%s).' % str(self.messageID)) with bsd.BsdTransaction(): locWordMetaData.WordMetaData.TransactionAwareCreate( wordPropertyID, self.messageID, metaDataValue, transactionBundle=transactionBundle)
def Create(cls, label, groupID=None, text='', context=None): cls._ErrorIfInTransaction( 'Message Create will not run within Transaction. Use TransactionAwareCreate.' ) with bsd.BsdTransaction('Creating new message: %s' % label) as bsdTransaction: cls._TransactionAwareCreate(label, groupID, LOCALE_SHORT_ENGLISH, text, context, wordTypeID=None, transactionBundle=None) resultList = bsdTransaction.GetTransactionResult() return cls.Get(resultList[0][1].messageID)
def _DeleteTags(self): batchSize = 1000 rowIndex = 0 projectToGroupsRows = Project._projectsToGroups.GetRows( projectID=self.projectID, _getDeleted=False) while rowIndex < len(projectToGroupsRows): with bsd.BsdTransaction(): for aTag in projectToGroupsRows[rowIndex:rowIndex + batchSize]: rowIndex += 1 aTag.Delete() projectsToLanguages = sm.GetService('bsdTable').GetTable( localizationBSDConst.PROJECT_LANGUAGE_TABLE) for aTag in projectsToLanguages.GetRows(projectID=self.projectID, _getDeleted=False): aTag.Delete()
def _DeleteChildren(self): with bsd.BsdTransaction(): childGroups = MessageGroup.GetMessageGroupsByParentID(self.groupID) for group in childGroups: if not group.Delete(): return False messages = locMessage.Message.GetMessagesByGroupID(self.groupID) for message in messages: if not message.Delete(): return False for aProject in locProject.Project.GetProjectsForGroup( self.groupID): aProject.RemoveGroupFromProject(self.groupID) return True
def _Copy(self, destGroupID, groupName): if sm.GetService('BSD').TransactionOngoing(): raise AuthoringValidationError( 'You cannot copy groups from within a transaction.') groupID = MessageGroup.Create(parentID=destGroupID, groupName=groupName, isReadOnly=self.isReadOnly, wordTypeID=self.wordTypeID).groupID with bsd.BsdTransaction( "Copying messages from group '%s' (groupID %s) to %s (groupID %s)" % (self.groupName, self.groupID, groupName, groupID)): for message in locMessage.Message.GetMessagesByGroupID( self.groupID): message.TransactionAwareCopy(groupID) childGroups = MessageGroup.GetMessageGroupsByParentID(self.groupID) for group in childGroups: group._Copy(groupID, group.groupName)
def _DeleteTags(self): """ Delete children records in batches. Cant do this in a single transaction because the string will become too large for some cases. """ batchSize = 1000 rowIndex = 0 projectToGroupsRows = Project._projectsToGroups.GetRows( projectID=self.projectID, _getDeleted=False) while rowIndex < len(projectToGroupsRows): with bsd.BsdTransaction(): for aTag in projectToGroupsRows[rowIndex:rowIndex + batchSize]: rowIndex += 1 aTag.Delete() projectsToLanguages = sm.GetService('bsdTable').GetTable( localizationBSDConst.PROJECT_LANGUAGE_TABLE) for aTag in projectsToLanguages.GetRows(projectID=self.projectID, _getDeleted=False): aTag.Delete()
def TransactionAwareCreate(cls, label, groupID=None, text='', context=None, transactionBundle=None): cls._ErrorIfNotInTransaction( 'Message TransactionAwareCreate will not run within Transaction. Use Create.' ) with bsd.BsdTransaction('Creating new message: %s' % label): actionIDsResult = cls._TransactionAwareCreate( label, groupID, LOCALE_SHORT_ENGLISH, text, context, wordTypeID=None, transactionBundle=transactionBundle) return actionIDsResult
def AddMetaDataEntryByName(self, propertyName, languageID, metaDataValue, transactionBundle=None): if self.wordTypeID == None: raise AuthoringValidationError( 'Before adding metadata, the wordType needs to be set on this messageID (%s).' % str(self.messageID)) typeRow = locWordType.WordType.Get(self.wordTypeID) if typeRow == None: raise AuthoringValidationError( 'WordTypeID (%s), of this message, does not exist.' % self.wordTypeID) typeName = typeRow.typeName with bsd.BsdTransaction(): locWordMetaData.WordMetaData.TransactionAwareCreateFromPropertyName( typeName, propertyName, languageID, self.messageID, metaDataValue, transactionBundle)
def TransactionAwareCreate(cls, label, groupID=None, text='', context=None, transactionBundle=None): """ This is the transaction-aware portion of the create code. It is split up from Create() on purpose; in order to allow other Wrappers to include these operations as port of their transaction as well. Purpose: Creates a new message label, and a blank English string by default. parameters: label - label of the message. Must be unique groupID - destination group. Must match the wordType of this message text - english text context - description for this message entry wordTypeID - type of the message transactionBundle - cache containing entries of messages to be added within transaction. It is required for validations within transactions. See CreateMessageDataBundle() pre-req: this IS meant to be called within nested Transactions. Returns: reserved actionID dictionary for the message, that will be added when transaction is done. The return is formated as: {"reservedMessageID": INTEGER} Notice: when things in base fail, this may return None """ cls._ErrorIfNotInTransaction( 'Message TransactionAwareCreate will not run within Transaction. Use Create.' ) with bsd.BsdTransaction('Creating new message: %s' % label): actionIDsResult = cls._TransactionAwareCreate( label, groupID, LOCALE_SHORT_ENGLISH, text, context, wordTypeID=None, transactionBundle=transactionBundle) return actionIDsResult
def _CopyAllMetaDataToNewMessage(cls, sourceMessageID, destMessageID, destinationWordTypeID, transactionBundle=None): cls.CheckAndSetCache() sourceMetaRows = cls._metaDataTable.GetRows(messageID=sourceMessageID, _getDeleted=False) with bsd.BsdTransaction(): for aMetaRow in sourceMetaRows: propertyRow = cls._propertyTable.GetRowByKey( aMetaRow.wordPropertyID) if propertyRow.wordTypeID != destinationWordTypeID: raise AuthoringValidationError( 'Source metadata property typeID doesnt match destination typeID. (%s, %s).' % (propertyRow.wordTypeID, destinationWordTypeID)) cls._TransactionAwareCreate( wordPropertyID=aMetaRow.wordPropertyID, messageID=destMessageID, metaDataValue=aMetaRow.metaDataValue, transactionBundle=transactionBundle) return True
def AddMetaDataEntry(self, wordPropertyID, metaDataValue, transactionBundle=None): """ Add new metadata entry to this message. Will throw an error if the metadata entry already exists. NOTE: This code is bsd transaction friendly. parameters: wordPropertyID - unique identifier for the property metaDataValue - metadata string transactionBundle - cache containing entries of messages to be added within transaction. It is required for validations within transactions. See CreateMessageDataBundle() """ if self.wordTypeID == None: raise AuthoringValidationError( 'Before adding metadata, the wordType needs to be set on this messageID (%s).' % str(self.messageID)) with bsd.BsdTransaction(): locWordMetaData.WordMetaData.TransactionAwareCreate( wordPropertyID, self.messageID, metaDataValue, transactionBundle=transactionBundle)
def _ImportXMLIntoDatabase(textRootElement, importBatchSize): import bsd dbzlocalization = sm.GetService('DB2').GetSchema('zlocalization') updatedTextEntries = { ADDED: 0, UPDATED: 0, UNCHANGED: 0, SKIPPED: 0, EMPTY_TAG: 0 } updatedTextEntriesDetail = {UPDATED: [], SKIPPED: [], EMPTY_TAG: []} updatedMetaDataEntries = {ADDED: 0, UPDATED: 0, UNCHANGED: 0, EMPTY_TAG: 0} updatedMetaDataEntriesDetail = {UPDATED: [], EMPTY_TAG: []} bsdTableService = sm.GetService('bsdTable') messageTextsTable = bsdTableService.GetTable( localizationBSDConst.MESSAGE_TEXTS_TABLE) metaDataTable = bsdTableService.GetTable( localizationBSDConst.WORD_METADATA_TABLE) messagesTable = bsdTableService.GetTable( localizationBSDConst.MESSAGES_TABLE) getSubmittedOnly = 1 allPropertiesResultSet = dbzlocalization.WordTypes_GetAllProperties( getSubmittedOnly, None) updatesResults = _GetUpdatesListForImportIntoDatabase( textRootElement, allPropertiesResultSet, messagesTable, importEnglishText) listOfTextImports, listOfMetaDataImports, textEntriesStats, metaDataEntriesStats, textEntriesDetails, metaDataEntriesDetails = updatesResults updatedTextEntries.update(textEntriesStats) updatedMetaDataEntries.update(metaDataEntriesStats) updatedTextEntriesDetail.update(textEntriesDetails) updatedMetaDataEntriesDetail.update(metaDataEntriesDetails) batchList = listOfTextImports[0:importBatchSize] batchIteration = 0 while batchList: with bsd.BsdTransaction(): for anEntry in batchList: messageID, translatedText, languageID, sourceDataID = anEntry updateStatus = _ImportTextEntry(messageID, translatedText, languageID, sourceDataID, messageTextsTable, compareTranslationWithEnglish) updatedTextEntries[updateStatus] += 1 if updateStatus == UPDATED: updatedTextEntriesDetail[updateStatus].append( (messageID, languageID)) batchIteration += 1 batchList = listOfTextImports[0 + batchIteration * importBatchSize:(1 + batchIteration) * importBatchSize] batchList = listOfMetaDataImports[0:importBatchSize] batchIteration = 0 while batchList: with bsd.BsdTransaction(): for anEntry in batchList: messageID, wordPropertyID, metaDataText = anEntry updateStatus = _ImportMetaDataEntry(messageID, wordPropertyID, metaDataText, metaDataTable) updatedMetaDataEntries[updateStatus] += 1 if updateStatus == UPDATED: updatedMetaDataEntriesDetail[updateStatus].append( (messageID, wordPropertyID)) batchIteration += 1 batchList = listOfMetaDataImports[0 + batchIteration * importBatchSize: (1 + batchIteration) * importBatchSize] return (updatedTextEntries, updatedMetaDataEntries, updatedTextEntriesDetail, updatedMetaDataEntriesDetail)
def ResetWordType(self): if self.wordTypeID is not None: with bsd.BsdTransaction(): self._DeleteMetaData() bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def _DeleteChildren(self): with bsd.BsdTransaction('Deleting message: %s' % self.label): self._DeleteText() self._DeleteMetaData() return True
def _ImportXMLIntoDatabase(textRootElement, importBatchSize): """ Imports translated text data and metadata from the XML string and updates the Database. parameters: textRootElement - root element pointing to the XML structure returns: tuple of two dictionaries specifying how many records were updated/added/unchanged or skipped, in an error case. First dictionary lists updates to text entries; second - to meta data entries exceptions: will throw various exceptions if XML is not correctly formatted prereq: Import must not run within a bsd Transaction. """ import bsd dbzlocalization = sm.GetService('DB2').GetSchema('zlocalization') updatedTextEntries = {ADDED: 0, UPDATED: 0, UNCHANGED: 0, SKIPPED: 0, EMPTY_TAG: 0} updatedTextEntriesDetail = {UPDATED: [], SKIPPED: [], EMPTY_TAG: []} updatedMetaDataEntries = {ADDED: 0, UPDATED: 0, UNCHANGED: 0, EMPTY_TAG: 0} updatedMetaDataEntriesDetail = {UPDATED: [], EMPTY_TAG: []} bsdTableService = sm.GetService('bsdTable') messageTextsTable = bsdTableService.GetTable(localizationBSDConst.MESSAGE_TEXTS_TABLE) metaDataTable = bsdTableService.GetTable(localizationBSDConst.WORD_METADATA_TABLE) messagesTable = bsdTableService.GetTable(localizationBSDConst.MESSAGES_TABLE) getSubmittedOnly = 1 allPropertiesResultSet = dbzlocalization.WordTypes_GetAllProperties(getSubmittedOnly, None) updatesResults = _GetUpdatesListForImportIntoDatabase(textRootElement, allPropertiesResultSet, messagesTable, importEnglishText) listOfTextImports, listOfMetaDataImports, textEntriesStats, metaDataEntriesStats, textEntriesDetails, metaDataEntriesDetails = updatesResults updatedTextEntries.update(textEntriesStats) updatedMetaDataEntries.update(metaDataEntriesStats) updatedTextEntriesDetail.update(textEntriesDetails) updatedMetaDataEntriesDetail.update(metaDataEntriesDetails) batchList = listOfTextImports[0:importBatchSize] batchIteration = 0 while batchList: with bsd.BsdTransaction(): for anEntry in batchList: messageID, translatedText, languageID, sourceDataID = anEntry updateStatus = _ImportTextEntry(messageID, translatedText, languageID, sourceDataID, messageTextsTable, compareTranslationWithEnglish) updatedTextEntries[updateStatus] += 1 if updateStatus == UPDATED: updatedTextEntriesDetail[updateStatus].append((messageID, languageID)) batchIteration += 1 batchList = listOfTextImports[0 + batchIteration * importBatchSize:(1 + batchIteration) * importBatchSize] batchList = listOfMetaDataImports[0:importBatchSize] batchIteration = 0 while batchList: with bsd.BsdTransaction(): for anEntry in batchList: messageID, wordPropertyID, metaDataText = anEntry updateStatus = _ImportMetaDataEntry(messageID, wordPropertyID, metaDataText, metaDataTable) updatedMetaDataEntries[updateStatus] += 1 if updateStatus == UPDATED: updatedMetaDataEntriesDetail[updateStatus].append((messageID, wordPropertyID)) batchIteration += 1 batchList = listOfMetaDataImports[0 + batchIteration * importBatchSize:(1 + batchIteration) * importBatchSize] return (updatedTextEntries, updatedMetaDataEntries, updatedTextEntriesDetail, updatedMetaDataEntriesDetail)
def SyncDataFromRegisteredTables(listOfRegistered = None, syncBatchSize = 1000): import bsd bsdTableService = sm.GetService('bsdTable') bsdService = sm.GetService('BSD') dbzlocalization = sm.GetService('DB2').GetSchema('zlocalization') groupTable = bsdTableService.GetTable(localization.MESSAGE_GROUPS_TABLE) messagesTable = bsdTableService.GetTable(localization.MESSAGES_TABLE) messageTextsTable = bsdTableService.GetTable(localization.MESSAGE_TEXTS_TABLE) keyToMessageMappingsTable = bsdTableService.GetTable(localization.MESSAGE_MAPPING_TABLE) localization.LogInfo('Loading BSD table information...') messagesTable.WaitForColumnsAllLoaded() messageTextsTable.WaitForColumnsAllLoaded() keyToMessageMappingsTable.WaitForColumnsAllLoaded() localization.LogInfo('BSD table information loaded.') if not bsd.login.IsLoggedIn(): localization.LogError('localizationSyncUtil: Cannot sync because we are not logged into BSD.') return allRegistered = [] if not listOfRegistered: getSubmittedOnly = 1 allRegistered = dbzlocalization.RegisteredResources_Select(getSubmittedOnly) else: regTable = bsdTableService.GetTable(localization.EXTERNAL_REGISTRATION_TABLE) for aResourceID in listOfRegistered: resourceRow = regTable.GetRowByKey(keyId1=aResourceID, _getDeleted=False) if resourceRow: allRegistered.append(resourceRow) updatedEntriesPerTable = {} localization.LogInfo('Beginning sync operation. Processing %s registered tables...' % len(allRegistered)) bsdUserID = bsd.login.GetCurrentUserId() contentServerName = localization.CONTENT_DATABASE_MAP.get(_GetCurrentDatabase(), 'localhost') currentReleaseID = None currentReleaseName = 'None' currentLinkType = None currentLinkID = None currentChangeID = 0 changelistDescription = None changeIDsInDescription = [] deletionChangeTextAdded = False descriptionOfOriginalChangelists = [] for registeredRowIndex, aRow in enumerate(allRegistered): localization.LogInfo('Processing table %s of %s...' % (registeredRowIndex + 1, len(allRegistered))) tableRegID = aRow.tableRegID textColumnName = aRow.columnRegName uniqueIDName = aRow.uniqueIDName groupPathName = aRow.groupPathName groupID = aRow.groupID groupRow = groupTable.GetRowByKey(keyId1=groupID) wordTypeID = groupRow.wordTypeID groupRow.isReadOnly = 1 registeredTypeID = aRow.registeredTypeID tableSchema = aRow.schemaRegName tableName = aRow.tableRegName fullLocationName = tableSchema + '.' + tableName + '.' + textColumnName syncResult = defaultdict(int) updatedEntriesPerTable['.'.join((aRow.schemaRegName, aRow.tableRegName, aRow.columnRegName))] = syncResult allTableData = None if registeredTypeID == localization.REGISTERED_TYPE_DYNAMIC: try: allTableData = dbzlocalization.RegisteredResources_SelectOnRegisteredResource(tableRegID) except SQLError: localization.LogError("failed when attempting to read a table/view resource : tableRegID = '%s'" % tableRegID) elif registeredTypeID == localization.REGISTERED_TYPE_BSD: try: registeredTable = bsdTableService.GetTable(tableSchema + '.' + tableName) allTableData = registeredTable.GetRows(_getDeleted=False) except RuntimeError: localization.LogError("failed when attempting to read a BSD table : '%s'" % (tableSchema + '.' + tableName)) else: localization.LogError("encountered unknown registered resource type : '%s' of type '%s'" % (tableSchema + '.' + tableName, registeredTypeID)) if allTableData is not None: sourceChangelists = {None: util.KeyVal(userID=None, userName='******', releaseID=None, releaseName='None', linkType=None, linkID=None)} cachedGroupPaths = {} possibleOrphanedGroups = set() updatedMappedEntries = [] localization.LogInfo('Fetching group and changelist information...') query = 'SELECT DISTINCT REV.changeID, C.userID, U.userName, C.releaseID, R.releaseName, C.linkType, C.linkID FROM zstatic.changes C\n INNER JOIN zstatic.revisions REV on REV.changeID = C.changeID\n INNER JOIN %s T on T.revisionID = REV.revisionID\n INNER JOIN zstatic.users U ON C.userID = U.userID\n LEFT JOIN zstatic.releases R on C.releaseID = R.releaseID\n WHERE REV.submitDate is not NULL' % (tableSchema + '.' + tableName) try: rs = sm.GetService('DB2').SQL(query) for row in rs: sourceChangelists[row.changeID] = row except SQLError as e: pass for aTableRow in allTableData: keyIDValue = getattr(aTableRow, uniqueIDName, None) if registeredTypeID == localization.REGISTERED_TYPE_DYNAMIC and groupPathName is not None: groupPathString = getattr(aTableRow, groupPathName, None) if groupPathString and groupPathString not in cachedGroupPaths: groupID, isNewGroup = _ResolveDynamicPath(groupPathString, aRow.groupID, wordTypeID) cachedGroupPaths[groupPathString] = groupID if isNewGroup == True: mappingEntry = localization.KeyToMessageMapping.GetMapping(keyIDValue, tableRegID) if mappingEntry is not None: messageEntry = localization.Message.GetMessageByID(mappingEntry.messageID) if messageEntry is not None: possibleOrphanedGroups.add(messageEntry.groupID) localization.LogInfo("Processing %s rows from target '%s'..." % (len(allTableData), fullLocationName)) rowIndex = 0 while rowIndex < len(allTableData): with bsd.BsdTransaction(): transactionBundle = localization.Message.CreateMessageDataBundle() for aTableRow in allTableData[rowIndex:rowIndex + syncBatchSize]: if registeredTypeID == localization.REGISTERED_TYPE_DYNAMIC and groupPathName is not None: groupPathString = getattr(aTableRow, groupPathName, None) if groupPathString is not None and groupPathString != '': groupID = cachedGroupPaths[groupPathString] else: groupID = aRow.groupID englishText = getattr(aTableRow, textColumnName, None) keyIDValue = getattr(aTableRow, uniqueIDName, None) descriptionText = '%s.%s.%s: %s' % (tableSchema, tableName, textColumnName, str(keyIDValue)) try: unicode(englishText) except UnicodeDecodeError as e: localization.LogError('Invalid text data in row, skipping entry:', aTableRow) syncResult[localization.KeyToMessageMapping.UNCHANGED_ENTRY] += 1 rowIndex += 1 continue updateAction = localization.KeyToMessageMapping.AddOrUpdateEntry(keyID=keyIDValue, tableRegID=tableRegID, groupID=groupID, label=None, englishText=englishText, descriptionText=descriptionText, transactionBundle=transactionBundle, returnOnly=True) if updateAction not in (localization.KeyToMessageMapping.UNCHANGED_ENTRY, localization.KeyToMessageMapping.SKIPPED_ENTRY): sourceReleaseID = sourceChangelists[aTableRow.changeID].releaseID sourceLinkType = sourceChangelists[aTableRow.changeID].linkType sourceLinkID = sourceChangelists[aTableRow.changeID].linkID if not currentChangeID or sourceReleaseID != currentReleaseID or sourceLinkType != currentLinkType or sourceLinkID != currentLinkID: localization.LogInfo("Row %s (of %s) was submitted under release '%s' with linkType '%s' and linkID '%s', but the current changelist is for release '%s' with linkType '%s' and linkID '%s'. Creating a new changelist and starting a new batch from this point..." % (rowIndex + 1, len(allTableData), sourceChangelists[aTableRow.changeID].releaseName, sourceLinkType, sourceLinkID, currentReleaseName, currentLinkType, currentLinkID)) changelistDescription = "Migrating external localizable content from '%s' to localization system tables. The original content was created or modified in release '%s' in the following BSD changelists:\n" % (fullLocationName, sourceChangelists[aTableRow.changeID].releaseName) changeIDsInDescription = [] currentReleaseID = sourceReleaseID currentReleaseName = sourceChangelists[aTableRow.changeID].releaseName currentLinkType = sourceLinkType currentLinkID = sourceLinkID currentChangeID = bsdService.ChangeAdd(bsdUserID, changeText=changelistDescription, linkType=currentLinkType, linkID=currentLinkID, releaseID=currentReleaseID) deletionChangeTextAdded = False localization.KeyToMessageMapping.AddOrUpdateEntry(keyID=keyIDValue, tableRegID=tableRegID, groupID=groupID, label=None, englishText=englishText, descriptionText=descriptionText, transactionBundle=transactionBundle) if aTableRow.changeID not in changeIDsInDescription: if aTableRow.changeID: descriptionOfOriginalChange = 'Change %s by %s (http://%s:50001/gd/bsd.py?action=Change&changeID=%s)\n' % (aTableRow.changeID, sourceChangelists[aTableRow.changeID].userName, contentServerName, aTableRow.changeID) changelistDescription += descriptionOfOriginalChange descriptionOfOriginalChangelists.append(descriptionOfOriginalChange) else: descriptionOfOriginalChange = 'Change by unknown user (' + fullLocationName + ' is not registered in BSD)\n' changelistDescription += descriptionOfOriginalChange descriptionOfOriginalChangelists.append(descriptionOfOriginalChange) changeIDsInDescription.append(aTableRow.changeID) bsdService.ChangeEdit(bsdUserID, currentChangeID, changelistDescription, linkType=currentLinkType, linkID=currentLinkID, releaseID=currentReleaseID) syncResult[updateAction] += 1 if updateAction != localization.KeyToMessageMapping.SKIPPED_ENTRY: updatedMappedEntries.append(keyIDValue) rowIndex += 1 localization.LogInfo('Finished processing %s rows. Results: %s' % (len(allTableData), dict(syncResult))) mappingsForResource = localization.KeyToMessageMapping.GetMappingsForRegisteredResource(tableRegID) if len(updatedMappedEntries) < len(mappingsForResource): localization.LogInfo('Searching %s records for obsolete mappings and orphaned groups...' % len(mappingsForResource)) for aMappingEntry in mappingsForResource: keyIDValue = aMappingEntry.keyID if keyIDValue not in updatedMappedEntries: messageEntry = localization.Message.GetMessageByID(aMappingEntry.messageID) if messageEntry is not None and messageEntry.groupID != aRow.groupID: groupEntry = groupTable.GetRowByKey(keyId1=messageEntry.groupID) if groupEntry.isReadOnly: possibleOrphanedGroups.add(groupEntry.groupID) if not currentChangeID: changelistDescription = 'Automatic deletion of obsolete localization mapping entries and orphaned groups.\n' currentChangeID = bsdService.ChangeAdd(bsdUserID, changeText=changelistDescription, linkType=None, linkID=None, releaseID=None) currentReleaseID = None deletionChangeTextAdded = True elif currentChangeID and not deletionChangeTextAdded: changelistDescription += 'Additionally, automatically deleting obsolete localization mapping entries and orphaned groups.\n' bsdService.ChangeEdit(bsdUserID, currentChangeID, changelistDescription, linkType=currentLinkType, linkID=currentLinkID, releaseID=currentReleaseID) currentReleaseID = None deletionChangeTextAdded = True aMappingEntry.Delete() syncResult[localization.KeyToMessageMapping.DELETED_ENTRY] += 1 localization.LogInfo('Finished searching for obsolete mappings and orphaned groups. %s mapping entries were deleted. %s potentially orphaned groups were detected.' % (syncResult[localization.KeyToMessageMapping.DELETED_ENTRY], len(possibleOrphanedGroups))) with bsd.BsdTransaction(): i = 0 while i < MAX_GROUP_DEPTH: possibleOrphanedGroups = _DeleteOrphanedGroupsFrom(possibleOrphanedGroups, cachedGroupPaths.values(), messagesTable, groupTable) if len(possibleOrphanedGroups) == 0: break elif len(possibleOrphanedGroups) == 1: if aRow.groupID in possibleOrphanedGroups: break i = i + 1 return (updatedEntriesPerTable, descriptionOfOriginalChangelists)