def __setattr__(self, key, value):
if key == 'wordTypeID':
if self.wordTypeID:
wordType = locWordType.WordType.Get(self.wordTypeID)
wordTypeName = wordType.typeName if wordType else 'None'
raise AuthoringValidationError(
"Cannot change wordTypeID: Group '%s' (groupID %s) may contain metadata for wordType '%s'; call ResetWordType first to delete all metadata in this group and try again."
% (self.groupName, self.groupID, wordTypeName))
if locWordType.WordType.Get(value) == None:
raise AuthoringValidationError(
'WordTypeID (%s) does not exist.' % value)
with bsd.BsdTransaction():
for message in locMessage.Message.GetMessagesByGroupID(
self.groupID):
message.wordTypeID = value
bsdWrappers.BaseWrapper.__setattr__(self, key, value)
return
if key == 'parentID' and value is not None:
if not MessageGroup.Get(value):
raise AuthoringValidationError(
"Cannot set parentID: '%s' is not a valid groupID." %
value)
if self._IsSubGroup(value):
subGroup = MessageGroup.Get(value)
raise AuthoringValidationError(
"You cannot assign group '%s' as a child of group '%s' because it would create a circular reference."
% (self.groupName, subGroup.groupName))
bsdWrappers.BaseWrapper.__setattr__(self, key, value)
def ResetWordType(self):
with bsd.BsdTransaction():
for message in locMessage.Message.GetMessagesByGroupID(
self.groupID):
message.ResetWordType()
bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def _CopyAllMetaDataToNewMessage(cls,
sourceMessageID,
destMessageID,
destinationWordTypeID,
transactionBundle=None):
"""
Copy all metadata entries to a newly created message (that may not exist yet)
This method is internal for all purposes
prereq:
sourceMessageID must exist in BSD
return:
True on success
"""
cls.CheckAndSetCache()
sourceMetaRows = cls._metaDataTable.GetRows(messageID=sourceMessageID,
_getDeleted=False)
with bsd.BsdTransaction():
for aMetaRow in sourceMetaRows:
propertyRow = cls._propertyTable.GetRowByKey(
aMetaRow.wordPropertyID)
if propertyRow.wordTypeID != destinationWordTypeID:
raise AuthoringValidationError(
'Source metadata property typeID doesnt match destination typeID. (%s, %s).'
% (propertyRow.wordTypeID, destinationWordTypeID))
cls._TransactionAwareCreate(
wordPropertyID=aMetaRow.wordPropertyID,
messageID=destMessageID,
metaDataValue=aMetaRow.metaDataValue,
transactionBundle=transactionBundle)
return True
def AddMetaDataEntryByName(self,
propertyName,
languageID,
metaDataValue,
transactionBundle=None):
"""
Add new metadata entry to this message. Will throw an error if the metadata entry already exists.
NOTE: This code is bsd transaction friendly.
parameters:
propertyName - name of the property
languageID - id for the language. For example "en-us"
metaDataValue - metadata string
transactionBundle - cache containing entries of messages to be added within transaction.
It is required for validations within transactions. See CreateMessageDataBundle()
"""
if self.wordTypeID == None:
raise AuthoringValidationError(
'Before adding metadata, the wordType needs to be set on this messageID (%s).'
% str(self.messageID))
typeRow = locWordType.WordType.Get(self.wordTypeID)
if typeRow == None:
raise AuthoringValidationError(
'WordTypeID (%s), of this message, does not exist.' %
self.wordTypeID)
typeName = typeRow.typeName
with bsd.BsdTransaction():
locWordMetaData.WordMetaData.TransactionAwareCreateFromPropertyName(
typeName, propertyName, languageID, self.messageID,
metaDataValue, transactionBundle)
def Create(cls, label, groupID=None, text='', context=None):
"""
Creates a new message label, and a blank English string by default.
parameters:
label - label of the message. Must be unique
groupID - destination group. Must match the wordType of this message
text - english text
context - description for this message entry
pre-req:
this MUST NOT be called within nested Transactions. This not what this method for.
exceptions:
throws various exceptions on failures
returns:
row of the message that we just added
"""
cls._ErrorIfInTransaction(
'Message Create will not run within Transaction. Use TransactionAwareCreate.'
)
with bsd.BsdTransaction('Creating new message: %s' %
label) as bsdTransaction:
cls._TransactionAwareCreate(label,
groupID,
LOCALE_SHORT_ENGLISH,
text,
context,
wordTypeID=None,
transactionBundle=None)
resultList = bsdTransaction.GetTransactionResult()
return cls.Get(resultList[0][1].messageID)
def ResetWordType(self):
"""
Change the wordTypeID of this message to None. This operation will remove all metadata belonging to
this message.
NOTE: This code is bsd transaction friendly.
"""
if self.wordTypeID is not None:
with bsd.BsdTransaction():
self._DeleteMetaData()
bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def ResetWordType(self):
"""
Deletes all existing metadata on all messages in the group, sets their word type to None, and sets the word type of the group to None.
"""
with bsd.BsdTransaction():
for message in locMessage.Message.GetMessagesByGroupID(
self.groupID):
message.ResetWordType()
bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def AddMetaDataEntry(self,
wordPropertyID,
metaDataValue,
transactionBundle=None):
if self.wordTypeID == None:
raise AuthoringValidationError(
'Before adding metadata, the wordType needs to be set on this messageID (%s).'
% str(self.messageID))
with bsd.BsdTransaction():
locWordMetaData.WordMetaData.TransactionAwareCreate(
wordPropertyID,
self.messageID,
metaDataValue,
transactionBundle=transactionBundle)
def Create(cls, label, groupID=None, text='', context=None):
cls._ErrorIfInTransaction(
'Message Create will not run within Transaction. Use TransactionAwareCreate.'
)
with bsd.BsdTransaction('Creating new message: %s' %
label) as bsdTransaction:
cls._TransactionAwareCreate(label,
groupID,
LOCALE_SHORT_ENGLISH,
text,
context,
wordTypeID=None,
transactionBundle=None)
resultList = bsdTransaction.GetTransactionResult()
return cls.Get(resultList[0][1].messageID)
def _DeleteTags(self):
batchSize = 1000
rowIndex = 0
projectToGroupsRows = Project._projectsToGroups.GetRows(
projectID=self.projectID, _getDeleted=False)
while rowIndex < len(projectToGroupsRows):
with bsd.BsdTransaction():
for aTag in projectToGroupsRows[rowIndex:rowIndex + batchSize]:
rowIndex += 1
aTag.Delete()
projectsToLanguages = sm.GetService('bsdTable').GetTable(
localizationBSDConst.PROJECT_LANGUAGE_TABLE)
for aTag in projectsToLanguages.GetRows(projectID=self.projectID,
_getDeleted=False):
aTag.Delete()
def _DeleteChildren(self):
with bsd.BsdTransaction():
childGroups = MessageGroup.GetMessageGroupsByParentID(self.groupID)
for group in childGroups:
if not group.Delete():
return False
messages = locMessage.Message.GetMessagesByGroupID(self.groupID)
for message in messages:
if not message.Delete():
return False
for aProject in locProject.Project.GetProjectsForGroup(
self.groupID):
aProject.RemoveGroupFromProject(self.groupID)
return True
def _Copy(self, destGroupID, groupName):
if sm.GetService('BSD').TransactionOngoing():
raise AuthoringValidationError(
'You cannot copy groups from within a transaction.')
groupID = MessageGroup.Create(parentID=destGroupID,
groupName=groupName,
isReadOnly=self.isReadOnly,
wordTypeID=self.wordTypeID).groupID
with bsd.BsdTransaction(
"Copying messages from group '%s' (groupID %s) to %s (groupID %s)"
% (self.groupName, self.groupID, groupName, groupID)):
for message in locMessage.Message.GetMessagesByGroupID(
self.groupID):
message.TransactionAwareCopy(groupID)
childGroups = MessageGroup.GetMessageGroupsByParentID(self.groupID)
for group in childGroups:
group._Copy(groupID, group.groupName)
def _DeleteTags(self):
"""
Delete children records in batches. Cant do this in a single transaction because the string will become too large for some cases.
"""
batchSize = 1000
rowIndex = 0
projectToGroupsRows = Project._projectsToGroups.GetRows(
projectID=self.projectID, _getDeleted=False)
while rowIndex < len(projectToGroupsRows):
with bsd.BsdTransaction():
for aTag in projectToGroupsRows[rowIndex:rowIndex + batchSize]:
rowIndex += 1
aTag.Delete()
projectsToLanguages = sm.GetService('bsdTable').GetTable(
localizationBSDConst.PROJECT_LANGUAGE_TABLE)
for aTag in projectsToLanguages.GetRows(projectID=self.projectID,
_getDeleted=False):
aTag.Delete()
def TransactionAwareCreate(cls,
label,
groupID=None,
text='',
context=None,
transactionBundle=None):
cls._ErrorIfNotInTransaction(
'Message TransactionAwareCreate will not run within Transaction. Use Create.'
)
with bsd.BsdTransaction('Creating new message: %s' % label):
actionIDsResult = cls._TransactionAwareCreate(
label,
groupID,
LOCALE_SHORT_ENGLISH,
text,
context,
wordTypeID=None,
transactionBundle=transactionBundle)
return actionIDsResult
def AddMetaDataEntryByName(self,
propertyName,
languageID,
metaDataValue,
transactionBundle=None):
if self.wordTypeID == None:
raise AuthoringValidationError(
'Before adding metadata, the wordType needs to be set on this messageID (%s).'
% str(self.messageID))
typeRow = locWordType.WordType.Get(self.wordTypeID)
if typeRow == None:
raise AuthoringValidationError(
'WordTypeID (%s), of this message, does not exist.' %
self.wordTypeID)
typeName = typeRow.typeName
with bsd.BsdTransaction():
locWordMetaData.WordMetaData.TransactionAwareCreateFromPropertyName(
typeName, propertyName, languageID, self.messageID,
metaDataValue, transactionBundle)
def TransactionAwareCreate(cls,
label,
groupID=None,
text='',
context=None,
transactionBundle=None):
"""
This is the transaction-aware portion of the create code. It is split up from Create() on purpose; in order to
allow other Wrappers to include these operations as port of their transaction as well.
Purpose: Creates a new message label, and a blank English string by default.
parameters:
label - label of the message. Must be unique
groupID - destination group. Must match the wordType of this message
text - english text
context - description for this message entry
wordTypeID - type of the message
transactionBundle - cache containing entries of messages to be added within transaction.
It is required for validations within transactions. See CreateMessageDataBundle()
pre-req:
this IS meant to be called within nested Transactions.
Returns: reserved actionID dictionary for the message, that will be added when transaction is done.
The return is formated as:
{"reservedMessageID": INTEGER}
Notice: when things in base fail, this may return None
"""
cls._ErrorIfNotInTransaction(
'Message TransactionAwareCreate will not run within Transaction. Use Create.'
)
with bsd.BsdTransaction('Creating new message: %s' % label):
actionIDsResult = cls._TransactionAwareCreate(
label,
groupID,
LOCALE_SHORT_ENGLISH,
text,
context,
wordTypeID=None,
transactionBundle=transactionBundle)
return actionIDsResult
def _CopyAllMetaDataToNewMessage(cls,
sourceMessageID,
destMessageID,
destinationWordTypeID,
transactionBundle=None):
cls.CheckAndSetCache()
sourceMetaRows = cls._metaDataTable.GetRows(messageID=sourceMessageID,
_getDeleted=False)
with bsd.BsdTransaction():
for aMetaRow in sourceMetaRows:
propertyRow = cls._propertyTable.GetRowByKey(
aMetaRow.wordPropertyID)
if propertyRow.wordTypeID != destinationWordTypeID:
raise AuthoringValidationError(
'Source metadata property typeID doesnt match destination typeID. (%s, %s).'
% (propertyRow.wordTypeID, destinationWordTypeID))
cls._TransactionAwareCreate(
wordPropertyID=aMetaRow.wordPropertyID,
messageID=destMessageID,
metaDataValue=aMetaRow.metaDataValue,
transactionBundle=transactionBundle)
return True
def AddMetaDataEntry(self,
wordPropertyID,
metaDataValue,
transactionBundle=None):
"""
Add new metadata entry to this message. Will throw an error if the metadata entry already exists.
NOTE: This code is bsd transaction friendly.
parameters:
wordPropertyID - unique identifier for the property
metaDataValue - metadata string
transactionBundle - cache containing entries of messages to be added within transaction.
It is required for validations within transactions. See CreateMessageDataBundle()
"""
if self.wordTypeID == None:
raise AuthoringValidationError(
'Before adding metadata, the wordType needs to be set on this messageID (%s).'
% str(self.messageID))
with bsd.BsdTransaction():
locWordMetaData.WordMetaData.TransactionAwareCreate(
wordPropertyID,
self.messageID,
metaDataValue,
transactionBundle=transactionBundle)
def _ImportXMLIntoDatabase(textRootElement, importBatchSize):
import bsd
dbzlocalization = sm.GetService('DB2').GetSchema('zlocalization')
updatedTextEntries = {
ADDED: 0,
UPDATED: 0,
UNCHANGED: 0,
SKIPPED: 0,
EMPTY_TAG: 0
}
updatedTextEntriesDetail = {UPDATED: [], SKIPPED: [], EMPTY_TAG: []}
updatedMetaDataEntries = {ADDED: 0, UPDATED: 0, UNCHANGED: 0, EMPTY_TAG: 0}
updatedMetaDataEntriesDetail = {UPDATED: [], EMPTY_TAG: []}
bsdTableService = sm.GetService('bsdTable')
messageTextsTable = bsdTableService.GetTable(
localizationBSDConst.MESSAGE_TEXTS_TABLE)
metaDataTable = bsdTableService.GetTable(
localizationBSDConst.WORD_METADATA_TABLE)
messagesTable = bsdTableService.GetTable(
localizationBSDConst.MESSAGES_TABLE)
getSubmittedOnly = 1
allPropertiesResultSet = dbzlocalization.WordTypes_GetAllProperties(
getSubmittedOnly, None)
updatesResults = _GetUpdatesListForImportIntoDatabase(
textRootElement, allPropertiesResultSet, messagesTable,
importEnglishText)
listOfTextImports, listOfMetaDataImports, textEntriesStats, metaDataEntriesStats, textEntriesDetails, metaDataEntriesDetails = updatesResults
updatedTextEntries.update(textEntriesStats)
updatedMetaDataEntries.update(metaDataEntriesStats)
updatedTextEntriesDetail.update(textEntriesDetails)
updatedMetaDataEntriesDetail.update(metaDataEntriesDetails)
batchList = listOfTextImports[0:importBatchSize]
batchIteration = 0
while batchList:
with bsd.BsdTransaction():
for anEntry in batchList:
messageID, translatedText, languageID, sourceDataID = anEntry
updateStatus = _ImportTextEntry(messageID, translatedText,
languageID, sourceDataID,
messageTextsTable,
compareTranslationWithEnglish)
updatedTextEntries[updateStatus] += 1
if updateStatus == UPDATED:
updatedTextEntriesDetail[updateStatus].append(
(messageID, languageID))
batchIteration += 1
batchList = listOfTextImports[0 + batchIteration *
importBatchSize:(1 + batchIteration) *
importBatchSize]
batchList = listOfMetaDataImports[0:importBatchSize]
batchIteration = 0
while batchList:
with bsd.BsdTransaction():
for anEntry in batchList:
messageID, wordPropertyID, metaDataText = anEntry
updateStatus = _ImportMetaDataEntry(messageID, wordPropertyID,
metaDataText,
metaDataTable)
updatedMetaDataEntries[updateStatus] += 1
if updateStatus == UPDATED:
updatedMetaDataEntriesDetail[updateStatus].append(
(messageID, wordPropertyID))
batchIteration += 1
batchList = listOfMetaDataImports[0 + batchIteration * importBatchSize:
(1 + batchIteration) *
importBatchSize]
return (updatedTextEntries, updatedMetaDataEntries,
updatedTextEntriesDetail, updatedMetaDataEntriesDetail)
def ResetWordType(self):
if self.wordTypeID is not None:
with bsd.BsdTransaction():
self._DeleteMetaData()
bsdWrappers.BaseWrapper.__setattr__(self, 'wordTypeID', None)
def _DeleteChildren(self):
with bsd.BsdTransaction('Deleting message: %s' % self.label):
self._DeleteText()
self._DeleteMetaData()
return True
def _ImportXMLIntoDatabase(textRootElement, importBatchSize):
"""
Imports translated text data and metadata from the XML string and updates the Database.
parameters:
textRootElement - root element pointing to the XML structure
returns:
tuple of two dictionaries specifying how many records were updated/added/unchanged or
skipped, in an error case. First dictionary lists updates to text entries; second -
to meta data entries
exceptions:
will throw various exceptions if XML is not correctly formatted
prereq:
Import must not run within a bsd Transaction.
"""
import bsd
dbzlocalization = sm.GetService('DB2').GetSchema('zlocalization')
updatedTextEntries = {ADDED: 0,
UPDATED: 0,
UNCHANGED: 0,
SKIPPED: 0,
EMPTY_TAG: 0}
updatedTextEntriesDetail = {UPDATED: [],
SKIPPED: [],
EMPTY_TAG: []}
updatedMetaDataEntries = {ADDED: 0,
UPDATED: 0,
UNCHANGED: 0,
EMPTY_TAG: 0}
updatedMetaDataEntriesDetail = {UPDATED: [],
EMPTY_TAG: []}
bsdTableService = sm.GetService('bsdTable')
messageTextsTable = bsdTableService.GetTable(localizationBSDConst.MESSAGE_TEXTS_TABLE)
metaDataTable = bsdTableService.GetTable(localizationBSDConst.WORD_METADATA_TABLE)
messagesTable = bsdTableService.GetTable(localizationBSDConst.MESSAGES_TABLE)
getSubmittedOnly = 1
allPropertiesResultSet = dbzlocalization.WordTypes_GetAllProperties(getSubmittedOnly, None)
updatesResults = _GetUpdatesListForImportIntoDatabase(textRootElement, allPropertiesResultSet, messagesTable, importEnglishText)
listOfTextImports, listOfMetaDataImports, textEntriesStats, metaDataEntriesStats, textEntriesDetails, metaDataEntriesDetails = updatesResults
updatedTextEntries.update(textEntriesStats)
updatedMetaDataEntries.update(metaDataEntriesStats)
updatedTextEntriesDetail.update(textEntriesDetails)
updatedMetaDataEntriesDetail.update(metaDataEntriesDetails)
batchList = listOfTextImports[0:importBatchSize]
batchIteration = 0
while batchList:
with bsd.BsdTransaction():
for anEntry in batchList:
messageID, translatedText, languageID, sourceDataID = anEntry
updateStatus = _ImportTextEntry(messageID, translatedText, languageID, sourceDataID, messageTextsTable, compareTranslationWithEnglish)
updatedTextEntries[updateStatus] += 1
if updateStatus == UPDATED:
updatedTextEntriesDetail[updateStatus].append((messageID, languageID))
batchIteration += 1
batchList = listOfTextImports[0 + batchIteration * importBatchSize:(1 + batchIteration) * importBatchSize]
batchList = listOfMetaDataImports[0:importBatchSize]
batchIteration = 0
while batchList:
with bsd.BsdTransaction():
for anEntry in batchList:
messageID, wordPropertyID, metaDataText = anEntry
updateStatus = _ImportMetaDataEntry(messageID, wordPropertyID, metaDataText, metaDataTable)
updatedMetaDataEntries[updateStatus] += 1
if updateStatus == UPDATED:
updatedMetaDataEntriesDetail[updateStatus].append((messageID, wordPropertyID))
batchIteration += 1
batchList = listOfMetaDataImports[0 + batchIteration * importBatchSize:(1 + batchIteration) * importBatchSize]
return (updatedTextEntries,
updatedMetaDataEntries,
updatedTextEntriesDetail,
updatedMetaDataEntriesDetail)
def SyncDataFromRegisteredTables(listOfRegistered = None, syncBatchSize = 1000):
import bsd
bsdTableService = sm.GetService('bsdTable')
bsdService = sm.GetService('BSD')
dbzlocalization = sm.GetService('DB2').GetSchema('zlocalization')
groupTable = bsdTableService.GetTable(localization.MESSAGE_GROUPS_TABLE)
messagesTable = bsdTableService.GetTable(localization.MESSAGES_TABLE)
messageTextsTable = bsdTableService.GetTable(localization.MESSAGE_TEXTS_TABLE)
keyToMessageMappingsTable = bsdTableService.GetTable(localization.MESSAGE_MAPPING_TABLE)
localization.LogInfo('Loading BSD table information...')
messagesTable.WaitForColumnsAllLoaded()
messageTextsTable.WaitForColumnsAllLoaded()
keyToMessageMappingsTable.WaitForColumnsAllLoaded()
localization.LogInfo('BSD table information loaded.')
if not bsd.login.IsLoggedIn():
localization.LogError('localizationSyncUtil: Cannot sync because we are not logged into BSD.')
return
allRegistered = []
if not listOfRegistered:
getSubmittedOnly = 1
allRegistered = dbzlocalization.RegisteredResources_Select(getSubmittedOnly)
else:
regTable = bsdTableService.GetTable(localization.EXTERNAL_REGISTRATION_TABLE)
for aResourceID in listOfRegistered:
resourceRow = regTable.GetRowByKey(keyId1=aResourceID, _getDeleted=False)
if resourceRow:
allRegistered.append(resourceRow)
updatedEntriesPerTable = {}
localization.LogInfo('Beginning sync operation. Processing %s registered tables...' % len(allRegistered))
bsdUserID = bsd.login.GetCurrentUserId()
contentServerName = localization.CONTENT_DATABASE_MAP.get(_GetCurrentDatabase(), 'localhost')
currentReleaseID = None
currentReleaseName = 'None'
currentLinkType = None
currentLinkID = None
currentChangeID = 0
changelistDescription = None
changeIDsInDescription = []
deletionChangeTextAdded = False
descriptionOfOriginalChangelists = []
for registeredRowIndex, aRow in enumerate(allRegistered):
localization.LogInfo('Processing table %s of %s...' % (registeredRowIndex + 1, len(allRegistered)))
tableRegID = aRow.tableRegID
textColumnName = aRow.columnRegName
uniqueIDName = aRow.uniqueIDName
groupPathName = aRow.groupPathName
groupID = aRow.groupID
groupRow = groupTable.GetRowByKey(keyId1=groupID)
wordTypeID = groupRow.wordTypeID
groupRow.isReadOnly = 1
registeredTypeID = aRow.registeredTypeID
tableSchema = aRow.schemaRegName
tableName = aRow.tableRegName
fullLocationName = tableSchema + '.' + tableName + '.' + textColumnName
syncResult = defaultdict(int)
updatedEntriesPerTable['.'.join((aRow.schemaRegName, aRow.tableRegName, aRow.columnRegName))] = syncResult
allTableData = None
if registeredTypeID == localization.REGISTERED_TYPE_DYNAMIC:
try:
allTableData = dbzlocalization.RegisteredResources_SelectOnRegisteredResource(tableRegID)
except SQLError:
localization.LogError("failed when attempting to read a table/view resource : tableRegID = '%s'" % tableRegID)
elif registeredTypeID == localization.REGISTERED_TYPE_BSD:
try:
registeredTable = bsdTableService.GetTable(tableSchema + '.' + tableName)
allTableData = registeredTable.GetRows(_getDeleted=False)
except RuntimeError:
localization.LogError("failed when attempting to read a BSD table : '%s'" % (tableSchema + '.' + tableName))
else:
localization.LogError("encountered unknown registered resource type : '%s' of type '%s'" % (tableSchema + '.' + tableName, registeredTypeID))
if allTableData is not None:
sourceChangelists = {None: util.KeyVal(userID=None, userName='******', releaseID=None, releaseName='None', linkType=None, linkID=None)}
cachedGroupPaths = {}
possibleOrphanedGroups = set()
updatedMappedEntries = []
localization.LogInfo('Fetching group and changelist information...')
query = 'SELECT DISTINCT REV.changeID, C.userID, U.userName, C.releaseID, R.releaseName, C.linkType, C.linkID FROM zstatic.changes C\n INNER JOIN zstatic.revisions REV on REV.changeID = C.changeID\n INNER JOIN %s T on T.revisionID = REV.revisionID\n INNER JOIN zstatic.users U ON C.userID = U.userID\n LEFT JOIN zstatic.releases R on C.releaseID = R.releaseID\n WHERE REV.submitDate is not NULL' % (tableSchema + '.' + tableName)
try:
rs = sm.GetService('DB2').SQL(query)
for row in rs:
sourceChangelists[row.changeID] = row
except SQLError as e:
pass
for aTableRow in allTableData:
keyIDValue = getattr(aTableRow, uniqueIDName, None)
if registeredTypeID == localization.REGISTERED_TYPE_DYNAMIC and groupPathName is not None:
groupPathString = getattr(aTableRow, groupPathName, None)
if groupPathString and groupPathString not in cachedGroupPaths:
groupID, isNewGroup = _ResolveDynamicPath(groupPathString, aRow.groupID, wordTypeID)
cachedGroupPaths[groupPathString] = groupID
if isNewGroup == True:
mappingEntry = localization.KeyToMessageMapping.GetMapping(keyIDValue, tableRegID)
if mappingEntry is not None:
messageEntry = localization.Message.GetMessageByID(mappingEntry.messageID)
if messageEntry is not None:
possibleOrphanedGroups.add(messageEntry.groupID)
localization.LogInfo("Processing %s rows from target '%s'..." % (len(allTableData), fullLocationName))
rowIndex = 0
while rowIndex < len(allTableData):
with bsd.BsdTransaction():
transactionBundle = localization.Message.CreateMessageDataBundle()
for aTableRow in allTableData[rowIndex:rowIndex + syncBatchSize]:
if registeredTypeID == localization.REGISTERED_TYPE_DYNAMIC and groupPathName is not None:
groupPathString = getattr(aTableRow, groupPathName, None)
if groupPathString is not None and groupPathString != '':
groupID = cachedGroupPaths[groupPathString]
else:
groupID = aRow.groupID
englishText = getattr(aTableRow, textColumnName, None)
keyIDValue = getattr(aTableRow, uniqueIDName, None)
descriptionText = '%s.%s.%s: %s' % (tableSchema,
tableName,
textColumnName,
str(keyIDValue))
try:
unicode(englishText)
except UnicodeDecodeError as e:
localization.LogError('Invalid text data in row, skipping entry:', aTableRow)
syncResult[localization.KeyToMessageMapping.UNCHANGED_ENTRY] += 1
rowIndex += 1
continue
updateAction = localization.KeyToMessageMapping.AddOrUpdateEntry(keyID=keyIDValue, tableRegID=tableRegID, groupID=groupID, label=None, englishText=englishText, descriptionText=descriptionText, transactionBundle=transactionBundle, returnOnly=True)
if updateAction not in (localization.KeyToMessageMapping.UNCHANGED_ENTRY, localization.KeyToMessageMapping.SKIPPED_ENTRY):
sourceReleaseID = sourceChangelists[aTableRow.changeID].releaseID
sourceLinkType = sourceChangelists[aTableRow.changeID].linkType
sourceLinkID = sourceChangelists[aTableRow.changeID].linkID
if not currentChangeID or sourceReleaseID != currentReleaseID or sourceLinkType != currentLinkType or sourceLinkID != currentLinkID:
localization.LogInfo("Row %s (of %s) was submitted under release '%s' with linkType '%s' and linkID '%s', but the current changelist is for release '%s' with linkType '%s' and linkID '%s'. Creating a new changelist and starting a new batch from this point..." % (rowIndex + 1,
len(allTableData),
sourceChangelists[aTableRow.changeID].releaseName,
sourceLinkType,
sourceLinkID,
currentReleaseName,
currentLinkType,
currentLinkID))
changelistDescription = "Migrating external localizable content from '%s' to localization system tables. The original content was created or modified in release '%s' in the following BSD changelists:\n" % (fullLocationName, sourceChangelists[aTableRow.changeID].releaseName)
changeIDsInDescription = []
currentReleaseID = sourceReleaseID
currentReleaseName = sourceChangelists[aTableRow.changeID].releaseName
currentLinkType = sourceLinkType
currentLinkID = sourceLinkID
currentChangeID = bsdService.ChangeAdd(bsdUserID, changeText=changelistDescription, linkType=currentLinkType, linkID=currentLinkID, releaseID=currentReleaseID)
deletionChangeTextAdded = False
localization.KeyToMessageMapping.AddOrUpdateEntry(keyID=keyIDValue, tableRegID=tableRegID, groupID=groupID, label=None, englishText=englishText, descriptionText=descriptionText, transactionBundle=transactionBundle)
if aTableRow.changeID not in changeIDsInDescription:
if aTableRow.changeID:
descriptionOfOriginalChange = 'Change %s by %s (http://%s:50001/gd/bsd.py?action=Change&changeID=%s)\n' % (aTableRow.changeID,
sourceChangelists[aTableRow.changeID].userName,
contentServerName,
aTableRow.changeID)
changelistDescription += descriptionOfOriginalChange
descriptionOfOriginalChangelists.append(descriptionOfOriginalChange)
else:
descriptionOfOriginalChange = 'Change by unknown user (' + fullLocationName + ' is not registered in BSD)\n'
changelistDescription += descriptionOfOriginalChange
descriptionOfOriginalChangelists.append(descriptionOfOriginalChange)
changeIDsInDescription.append(aTableRow.changeID)
bsdService.ChangeEdit(bsdUserID, currentChangeID, changelistDescription, linkType=currentLinkType, linkID=currentLinkID, releaseID=currentReleaseID)
syncResult[updateAction] += 1
if updateAction != localization.KeyToMessageMapping.SKIPPED_ENTRY:
updatedMappedEntries.append(keyIDValue)
rowIndex += 1
localization.LogInfo('Finished processing %s rows. Results: %s' % (len(allTableData), dict(syncResult)))
mappingsForResource = localization.KeyToMessageMapping.GetMappingsForRegisteredResource(tableRegID)
if len(updatedMappedEntries) < len(mappingsForResource):
localization.LogInfo('Searching %s records for obsolete mappings and orphaned groups...' % len(mappingsForResource))
for aMappingEntry in mappingsForResource:
keyIDValue = aMappingEntry.keyID
if keyIDValue not in updatedMappedEntries:
messageEntry = localization.Message.GetMessageByID(aMappingEntry.messageID)
if messageEntry is not None and messageEntry.groupID != aRow.groupID:
groupEntry = groupTable.GetRowByKey(keyId1=messageEntry.groupID)
if groupEntry.isReadOnly:
possibleOrphanedGroups.add(groupEntry.groupID)
if not currentChangeID:
changelistDescription = 'Automatic deletion of obsolete localization mapping entries and orphaned groups.\n'
currentChangeID = bsdService.ChangeAdd(bsdUserID, changeText=changelistDescription, linkType=None, linkID=None, releaseID=None)
currentReleaseID = None
deletionChangeTextAdded = True
elif currentChangeID and not deletionChangeTextAdded:
changelistDescription += 'Additionally, automatically deleting obsolete localization mapping entries and orphaned groups.\n'
bsdService.ChangeEdit(bsdUserID, currentChangeID, changelistDescription, linkType=currentLinkType, linkID=currentLinkID, releaseID=currentReleaseID)
currentReleaseID = None
deletionChangeTextAdded = True
aMappingEntry.Delete()
syncResult[localization.KeyToMessageMapping.DELETED_ENTRY] += 1
localization.LogInfo('Finished searching for obsolete mappings and orphaned groups. %s mapping entries were deleted. %s potentially orphaned groups were detected.' % (syncResult[localization.KeyToMessageMapping.DELETED_ENTRY], len(possibleOrphanedGroups)))
with bsd.BsdTransaction():
i = 0
while i < MAX_GROUP_DEPTH:
possibleOrphanedGroups = _DeleteOrphanedGroupsFrom(possibleOrphanedGroups, cachedGroupPaths.values(), messagesTable, groupTable)
if len(possibleOrphanedGroups) == 0:
break
elif len(possibleOrphanedGroups) == 1:
if aRow.groupID in possibleOrphanedGroups:
break
i = i + 1
return (updatedEntriesPerTable, descriptionOfOriginalChangelists)
