def __init__(self, path, primaryLangCode, prunepath=None, markconflicts=False):
"""Create a translation file at the specified path. If `mergeKey` is
not None, then the Translation file will do its comparisons based on
the language code (LCID) given; examples are '1033' or '2058'. This
means it will compare and combine on that language code. So if '1033'
was given all similar '1033' values for ALL STRINGS IN THE SYSTEM will
be pushed into one line.
"""
self.__path = path
self.__primaryLangCode = primaryLangCode
self.__markconflicts = markconflicts
self.__mergelist = {} #mid -> [id]
self.__strings = {} # mid -> RCStringValue => sheet1
self.__conflicts=[] # [mid] => sheet1 (if markConflicts is True)
self.__utils = {} # idn -> (order,type) => sheet2
self.__projs = {} # projname -> number => sheet2
self.__seps = [] # [ xpath-id ] => sheet2
self.__pruned = {} # mid -> RCStringValue => sheet2
self.__pruner = Pruner( prunepath )
class TranslationFile():
"""A Translation File is an Excel Workbook with two work-sheets. The
first worksheet is for Menus, Dialogs, and StringTables all merged together
and the second is utility information to be ignored by the translators
(there will be a message that says so). The main worksheet is a condensed
form of its parent utility files to save on translation costs. This means
that there will only be unique strings for each line.
On the first worksheet, the first column is a utility column that should
be ignored by the translators, it will be minimized and/or hidden to
reduce clutter. The first column is a list of strings for translation, and
they should just fill in the column next to it.
"""
SHEET_NAMES = { 0:"Strings",
1:"util" }
STRINGS_SHEET = SHEET_NAMES[0]
UTIL_SHEET = SHEET_NAMES[1]
HEADER_COLS = ['id']
UTIL_SHEET_WARNING = \
"""***DO NOT EDIT! THIS IS USED FOR PARSING THIS TRANSLATION FILE AFTERWARD.***"""
def __init__(self, path, primaryLangCode, prunepath=None, markconflicts=False):
"""Create a translation file at the specified path. If `mergeKey` is
not None, then the Translation file will do its comparisons based on
the language code (LCID) given; examples are '1033' or '2058'. This
means it will compare and combine on that language code. So if '1033'
was given all similar '1033' values for ALL STRINGS IN THE SYSTEM will
be pushed into one line.
"""
self.__path = path
self.__primaryLangCode = primaryLangCode
self.__markconflicts = markconflicts
self.__mergelist = {} #mid -> [id]
self.__strings = {} # mid -> RCStringValue => sheet1
self.__conflicts=[] # [mid] => sheet1 (if markConflicts is True)
self.__utils = {} # idn -> (order,type) => sheet2
self.__projs = {} # projname -> number => sheet2
self.__seps = [] # [ xpath-id ] => sheet2
self.__pruned = {} # mid -> RCStringValue => sheet2
self.__pruner = Pruner( prunepath )
def getPath(self):
"""Gets the path of the Translation File."""
return self.__path
def load(self, newpath=None):
"""Loads the translation file into memory for use. If `newpath` is
set, then the file is changed.
"""
if newpath is not None: self.__path = newpath
# Load the utils first:
self.__loadUtilLines()
# Then load the strings:
self.__loadStringLines()
return True
def save(self, langcodes=None, order=False, newpath=None, trim=False):
"""Saves the Translation file to the specified path, or the path
given to it upon instantiation.
"""
if newpath is not None: self.__path = newpath
workbook = xlwt.Workbook()
strings = workbook.add_sheet( self.STRINGS_SHEET )
util = workbook.add_sheet( self.UTIL_SHEET )
langcodes = self._makeLangList() if langcodes is None else langcodes
head = self._makeHeader(langcodes)
lines, conflicts = self.__getStrLines( langcodes, order, trim )
self.__writeLines( strings,
lines,
header=head,
conflictIndexs=conflicts,
headermagic=True)
self.__writeLines( util,
self.__getUtilLines(head, langcodes, order, trim),
header=[self.UTIL_SHEET_WARNING],
hideUtilCol=False)
workbook.save(self.__path)
def getSysMenuFile(self, path, save=False, langcodes=None):
""" Generates a System Menu File from the internals of the translation
file. This does not save the file, instead is merely returned. This can
be changed, by setting `save` to True.
"""
csvfile = self.getSysMenuFileCSV('tmp.cmenus', langcodes=langcodes)
return ConvertMenuCSV2XML(path, csvfile, autosave=save)
def getSysMenuFileCSV(self, path, save=False, langcodes=None):
"""Generates a System Menu CSV File from the internals of the translation
file. This does not save the file, instead is merely returned. This can
be changed, by setting `save` to True.
"""
global MenuIdMatcher
csvfile = SysMenuFileCSV(path)
data ={}
for mid, ids in self.__mergelist.items():
for idn in ids:
if MenuIdMatcher.search(idn) is not None:
newid = idn.split(".",1)[1]
try: data[newid] = self.__strings[mid].limitByCodes(langcodes)
except: data[newid] = self.__pruned[mid].limitByCodes(langcodes)
csvfile._setInternals(data, self.__projs, self.__utils, self.__seps)
if save: csvfile.save()
return csvfile
def getSysDialogFile(self, path, save=False, langcodes=None):
""" Generates a System Dialog File from the internals of the
translation file. This does not save the file, instead is merely
returned. This can be changed, by setting `save` to True.
"""
global DialogIdMatcher
dlog = SysDialogFile(path)
projdialogs = {} # projkey -> { dialogid -> dialog }
for mid, ids in self.__mergelist.items():
for idn in ids:
if DialogIdMatcher.search(idn) is not None:
projkey, did, strid = DialogIdMatcher.search(idn).groups()
try: val = copy.deepcopy( self.__strings[mid] )
except: val = copy.deepcopy( self.__pruned[mid] )
val.setID( strid )
val = val.limitByCodes( langcodes )
if projdialogs.get(projkey) is None:
projdialogs[projkey] = {}
if projdialogs[projkey].get(did, None) is None:
dialog = RCDialog(did)
dialog._values.append(val)
projdialogs[projkey][did] = dialog
else:
projdialogs[projkey][did]._values.append(val)
for projkey, dlogs in projdialogs.items():
dlog._projs[self.getProjName(projkey)] = dlogs.values()
if save: dlog.save()
return dlog
def getSysStrTblFile(self, path, save=False, langcodes=None):
""" Generates a System String Table File from the internals of the
translation file. This does not save the file, instead is merely
returned. This can be changed, by setting `save` to True.
"""
strtblfile = SysStrTblFile( path )
projtables = {} # projkey -> RCStrTbl
for mid, ids in self.__mergelist.items():
for idn in ids:
if ConstantIdMatcher.search(idn) is not None:
projkey, consid = ConstantIdMatcher.search(idn).groups()
try: val = copy.deepcopy( self.__strings[mid] )
except: val = copy.deepcopy( self.__pruned[mid] )
val.setID( consid )
if projtables.get(projkey, None) is None:
projtables[projkey] = RCStrTbl()
projtables[projkey].addStringValue( val.limitByCodes(langcodes) )
for proj, strtbl in projtables.items():
strtblfile._projs[ self.getProjName(proj) ] = strtbl
if save: strtblfile.save()
return strtblfile
def updateWithMenuFile(self, menuFile, preloaded=True, saveAfter=False):
"""Updates the translation file with the values of a system level
menu file. It will not save the translation file unless `saveAfter` is
True. The `preloaded` parameter is to know whether the file has been
loaded into memory already.
"""
csvmenu = ConvertMenuXML2CSV('tmp.cmenus', menuFile, preloaded)
dialogs = self.getSysDialogFile('tmp.dialogs')
strings = self.getSysStrTblFile('tmp.strtbls')
self.setSysFiles(csvmenu, dialogs, strings)
if saveAfter: self.save()
def updateWithDialogFile(self, dialogFile, preloaded=True, saveAfter=False):
"""Updates the translation file with the values of a system level
dialog file. It will not save the translation file unless `saveAfter`
is True. The `preloaded` parameter is to know whether the file has been
loaded into memory already.
"""
if not preloaded: dialogFile.load()
csvmenu = self.getSysMenuFileCSV('tmp.cmenus')
strings = self.getSysStrTblFile('tmp.strtbls')
self.setSysFiles(csvmenu, dialogFile, strings)
if saveAfter: self.save()
def updateWithStrTblFile(self, strFile, preloaded=True, saveAfter=False):
"""Updates the translation file with the values of a system level
string table file. It will not save the translation file unless
`saveAfter` is True. The `preloaded` parameter is to know whether the
file has been loaded into memory already.
"""
if not preloaded: strFile.load()
csvmenu = self.getSysMenuFileCSV('tmp.cmenus')
dialogs = self.getSysDialogFile('tmp.dialogs')
self.setSysFiles(csvmenu, dialogs, strFile)
if saveAfter: self.save()
def setSysFiles(self, csvmenu, dialogs, strings):
"""Adds the three files. Assume that this function will overwrite ALL
current data. It will not do any updating.
"""
self.__strings, self.__utils, self.__projs, self.__seps, self.__pruned = {}, {}, {}, [], {}
self.__projs, self.__utils, self.__seps = csvmenu._getXMLUtilSections()
data = csvmenu._getXMLDataSection()
for id, val in data.items(): #the csv makes it easy
newid = "m.%s"%id
self.__addStringLine(newid, val)
for project, table in strings._projs.items():
try:
if table is None: continue
for val in table._values:
newid = "c.%d.%s"%(self.getProjKey(project), str(val.getID()))
self.__addStringLine(newid, val)
except KeyError:
logging.error("Project %s doesn't exist in given string table file."%project)
for project, dlogs in dialogs._projs.items():
try:
for dlog in dlogs:
newid = "d.%d.%s"%( self.getProjKey(project), str(dlog.id))
for val in dlog._values:
vid = "%s.%s"%( newid, str(val.getID()))
self.__addStringLine(vid, val)
except KeyError:
logging.error("Project %s doesn't exist in given dialog file."%project)
def getProjKey(self, name):
"""Get the project number mapping for a project name."""
try:
return self.__projs[name]
except:
maxval = max(self.__projs.values())
self.__projs[name] = maxval+1
return maxval+1
def getProjName(self, num):
#logging.error("num = %s, projs= %s"% (num, self.__projs))
for k,v in self.__projs.items():
#logging.error("getting projname: %s ?= %s"%(v,num))
if v == num: return k
raise Exception("Couldnt find name!!")
def getProjNames(self):
""" Returns a list of all the projects in the translation file."""
return list(self.__projs.keys())
def _makeLangList(self):
lst = {}
for val in self.__strings.values():
for l in val.getLangCodes(): lst[l]=1
langorder = list(lst.keys())
return langorder
def _makeHeader(self, langcodes):
langs = []
for langcode in langcodes:
lang= ToLanguageString( int(langcode) )
if lang is None: langs.append(langcode)
else: langs.append( lang )
return TranslationFile.HEADER_COLS+langs
def __addStringLine(self, idn, value):
found = False
if not self.__pruner.isPrunable( value, self.__primaryLangCode ): #then add to strings dict
err = False
for mid, val in self.__strings.items():
comp,err = val.compare( value, ignoreID=True, retErr=True )
if err and self.__markconflicts:
self.__conflicts.append(mid)
if comp:
self.__mergelist[mid].append(idn)
self.__strings[mid].combine(value, True,True)
found = True
break
if not found:
mid = uuid.uuid4()
self.__mergelist[mid] = [idn]
if err and self.__markconflicts:
self.__conflicts.append(mid)
self.__strings[mid] = value
else: #since we can prune it, lets add it to the prune list.
for mid, val in self.__pruned.items():
if val.compare( value, ignoreID=True ) :
self.__mergelist[mid].append(idn)
self.__pruned[mid].combine(value, True,True)
found = True
break
if not found:
mid = uuid.uuid4()
self.__mergelist[mid] = [idn]
self.__pruned[mid] = value
def __translateCodes(self, langstrs):
codes = []
for langstr in langstrs:
code = ToLocalID(langstr)
if code is None:
logging.error("COULD NOT DETERMINE LOCAL ID FROM: %s"%langstr)
code = ''
codes.append(str(code))
return codes
def __loadStringLines(self):
global IDMatcher
rows = self.__readSheet( TranslationFile.STRINGS_SHEET, 0 )
langcodes = []
headeroffset = len(TranslationFile.HEADER_COLS)
for head in rows:
langcodes = head[headeroffset:]
break
langcodes = self.__translateCodes( langcodes )
for row in rows:
midlist=row[0].split(",")
col = 1
val = RCStringValue()
for lang in langcodes:
val.addValuePair(lang, row[col])
col+=1
#add value
for idn in midlist: self.__addStringLine(idn, val) #OMG SO SLOW, but it works
def __loadUtilLines(self):
rows = self.__readSheet( TranslationFile.UTIL_SHEET, 1 )
langcodes = []
headeroffset = len(TranslationFile.HEADER_COLS)
for head in rows:
langcodes = head[headeroffset:]
break
langcodes = self.__translateCodes( langcodes )
projOrUtil = 0 # 0=proj,1=util,2=seps,3=pruned
for row in rows:
if row[0] == "PROJMAP":
projOrUtil = 0
continue
elif row[0] == "XMLUTIL":
projOrUtil = 1
continue
elif row[0] == "SEPS":
projOrUtil = 2
continue
elif row[0] == "PRUNED":
projOrUtil = 3
continue
elif IDMatcher.search(row[0]) is not None:
if projOrUtil==1 and row[1].isdigit():
idn, order, typeid, *_ = row #ignore the man behind the curtain
self.__utils[idn] = (order, typeid)
elif projOrUtil==2:
self.__seps.append( row[0] )
elif projOrUtil==3:
midlist=row[0].split(",")
col = 1
val = RCStringValue()
for lang in langcodes:
val.addValuePair(lang, row[col])
col+=1
mid = uuid.uuid4()
self.__mergelist[mid]=midlist
self.__pruned[mid]=val
else:
#sometimes a project name might match an ID, in
# these instances we have to be careful.
if projOrUtil==0 and row[1].isdigit() and self.__rest(2, row, ''):
self.__projs[ row[0] ] = row[1]
else:
logging.debug("UNKNOWN UTIL LINE: %s, state=(%d)"%(row, projOrUtil))
continue #broken line?
else:
if projOrUtil==0 and row[1].isdigit():
self.__projs[ row[0] ] = row[1]
else:
logging.debug("UNKNOWN UTIL LINE: %s"%row)
continue #broken line?
def __rest(self, strtIndex, row, val):
for x in range(strtIndex, len(row)):
if row[x] != val: return False
return True
def __breakUpList(self, lst, maxsize=30000, fakeLimit=300, newlst=None):
if len(",".join(lst)) > maxsize:
size = int(len(lst)/2)
#break the list into equal parts
a,b = lst[:size], lst[size:]
#recurse on each part
alst = self.__breakUpList(a,newlst=newlst)
blst = self.__breakUpList(b,newlst=newlst)
#upon return, concat the two lists and return
return alst+blst
else:
if newlst is None: return [",".join(lst)]
else:
newlst.append([",".join(lst)])
return newlst
def __getUtilLines(self, header, langorder, order=False, trim=False):
lines = [header]
lines.append(['PROJMAP'])
for proj,num in self.__projs.items():
lines.append([proj, str(num)])
lines.append(['XMLUTIL'])
for idn, val in self.__utils.items():
order, typeid = val
lines.append([idn, order, typeid])
lines.append(["SEPS"])
for xpath in self.__seps:
lines.append([xpath])
lines.append(["PRUNED"])
key = None if not order else (lambda x: x[1].getValue(langorder[0],''))
for mid, val in sorted(self.__pruned.items(), key=key):
midlist = ",".join(self.__mergelist[mid])
if len(midlist) > 30000: #max cell size
midlines = self.__breakUpList(self.__mergelist[mid])
for line in midlines:
for lang in langorder:
line.append(val.getValue(lang,''))
if trim and self.__rest(1, line, ''): continue
lines.append(line)
else:
line = [midlist]
for lang in langorder:
line.append(val.getValue(lang,''))
if trim and self.__rest(1, line, ''): continue
lines.append(line)
return lines
def __getStrLines(self, langorder, order=False, trim=False):
lines = []
conflictIndexs = []
remove = len(langorder) <= 1
key = None if not order or len(langorder)==0 else (lambda x: x[1].getValue(langorder[0], ''))
for mid, val in sorted(self.__strings.items(), key=key):
midlist = ",".join(self.__mergelist[mid])
if len(midlist) > 30000: #max cell size
midlines = self.__breakUpList(self.__mergelist[mid])
for line in midlines:
for lang in langorder:
tmp = val.getValue(lang,'')
if remove and tmp == '': continue
line.append(tmp)
if trim and self.__rest(1, line, ''): continue
if self.__markconflicts and mid in self.__conflicts:
conflictIndexs.append(len(lines))
lines.append(line)
else:
line = [midlist]
for lang in langorder:
tmp = val.getValue(lang,'')
if remove and tmp == '': continue
line.append(tmp)
if trim and self.__rest(1, line, ''): continue
if self.__markconflicts and mid in self.__conflicts:
conflictIndexs.append(len(lines))
lines.append(line)
logging.debug("Unique Strings:%d, Possible Conflicts:%d"%(len(lines),len(conflictIndexs)))
return lines, conflictIndexs
def __readSheet(self, name, rowstart=0):
"""Reads through a sheet, row by row. Starting at the specified row.
The return will be a list of strings, similarly to the CSV package.
"""
workbook = xlrd.open_workbook(filename=self.__path, on_demand=True)
sheet = workbook.sheet_by_name(name)
for rownum in range(rowstart, sheet.nrows):
vals=[]
for cell in sheet.row(rownum):
if cell.ctype in [0,6]: #blank string
vals.append('')
elif cell.ctype == 2: #number
try:
val = float(cell.value)
vals.append(str(int(val)))
except: vals.append(str(cell.ctype))
else: #text,date,bool,error
vals.append(str(cell.value))
yield vals
def __writeLines(self, worksheet, lines, header=None, headermagic=False, hideUtilCol=True, conflictIndexs=None):
global CONFLICT_STYLE, NORMAL_STYLE, HEADER_STYLE
badex = [] if conflictIndexs is None else conflictIndexs
if header is not None:
col = 0
format = HEADER_STYLE if headermagic else NORMAL_STYLE
for val in header:
worksheet.write(0,col,val,format)
col+=1
if headermagic:
worksheet.set_panes_frozen(True) # frozen headings instead of split panes
worksheet.set_horz_split_pos(1) # in general, freeze after last heading row
worksheet.set_remove_splits(True) # if user does unfreeze, don't leave a split there
row = 0 if header is None else 1
curindex = 0
for line in lines:
if curindex in badex:
format = CONFLICT_STYLE
else: format = NORMAL_STYLE
col = 0
for val in line:
worksheet.write(row, col, val, format)
col+=1
row+=1
curindex+=1
if hideUtilCol: worksheet.col(0).width = 0x0