def closefile(mydict, lineno):
if nodes is not None:
if len(nodes) > 1:
err_message = ("End of WML file reached, but some tags were " +
"not properly closed.\n"
"(nearest unclosed tag is: " +
nodes[-1].tagname + ")" )
finfo = fileref + ":" + str(lineno)
wmlerr(finfo, err_message)
else:
_closenode_update_dict(mydict)
def closefile(mydict, lineno):
if nodes is not None:
if len(nodes) > 1:
err_message = ("End of WML file reached, but some tags were " +
"not properly closed.\n"
"(nearest unclosed tag is: " + nodes[-1].tagname +
")")
finfo = fileref + ":" + str(lineno)
wmlerr(finfo, err_message)
else:
_closenode_update_dict(mydict)
def closenode(closetag, mydict, lineno):
global fileref
global fileno
global nodes
if nodes is None:
err_message = ("unexpected closing tag '" +
closetag + "' outside any scope.")
finfo = fileref + ":" + str(lineno)
wmlerr(finfo, err_message)
else:
# node to close is the LAST element in self.nodes list
mytag = nodes[-1].tagname
mynode = nodes[-1]
expected_closetag = re.sub(r'\[', r'[/', mytag)
finfo = fileref + ":" + str(lineno)
if mynode.tagname == "":
err_message = ("unexpected closing tag '" +
closetag + "' outside any scope.")
wmlerr(finfo, err_message)
else:
if closetag != expected_closetag:
err_message = ("expected closing tag '" +
expected_closetag + "' but '" +
closetag + "' found.")
wmlerr(finfo, err_message)
_closenode_update_dict(mydict)
nodes.pop()
def closenode(closetag, mydict, lineno):
global fileref
global fileno
global nodes
if nodes is None:
err_message = ("unexpected closing tag '" + closetag +
"' outside any scope.")
finfo = fileref + ":" + str(lineno)
wmlerr(finfo, err_message)
else:
# node to close is the LAST element in self.nodes list
mytag = nodes[-1].tagname
mynode = nodes[-1]
expected_closetag = re.sub(r'\[', r'[/', mytag)
finfo = fileref + ":" + str(lineno)
if mynode.tagname == "":
err_message = ("unexpected closing tag '" + closetag +
"' outside any scope.")
wmlerr(finfo, err_message)
else:
if closetag != expected_closetag:
err_message = ("expected closing tag '" + expected_closetag +
"' but '" + closetag + "' found.")
wmlerr(finfo, err_message)
_closenode_update_dict(mydict)
nodes.pop()
def run(self, xline, lineno, match):
status = pywmlx.state.machine._pending_luastring.plural.status
realmatch = re.match(r'\s*(\S)', xline)
if realmatch:
_nextstate = 'lua_idle'
if realmatch.group(1) == '"':
_nextstate = 'lua_pl01'
elif realmatch.group(1) == "'":
_nextstate = 'lua_pl02'
elif realmatch.group(1) == '[':
_nextstate = 'lua_pl03'
elif realmatch.group(1) == ')':
if status == 'wait_string':
finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
errmsg = ("first argument of function '_ (...)' must be a "
"string")
wmlerr(finfo, errmsg)
else:
# if parenthesis is closed rightly after first argument
# this is not a plural form. So we set
# PendingLuaString.plural to None
if status == 'wait_plural':
pywmlx.state.machine._pending_luastring.plural = None
xline = xline [ realmatch.end(): ]
return (xline, 'lua_idle')
else:
# this should never happen. But if the first value after _ (
# does not start with a string marker, than wmlxgettext will
# finish with an error
finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
errmsg = "first argument of function '_ (...)' must be a string"
wmlerr(finfo, errmsg)
xline = xline [ realmatch.end(): ]
xline = realmatch.group(1) + xline
return (xline, _nextstate)
else:
# You can find a new line rigthly after _ (. In this case, we will
# continue to search a string into the next line
return (None, 'lua_plidle1')
def run(self, xline, lineno, match):
status = pywmlx.state.machine._pending_luastring.plural.status
realmatch = re.match(r'\s*(\S)', xline)
if realmatch:
_nextstate = 'lua_idle'
if realmatch.group(1) == '"':
_nextstate = 'lua_pl01'
elif realmatch.group(1) == "'":
_nextstate = 'lua_pl02'
elif realmatch.group(1) == '[':
_nextstate = 'lua_pl03'
elif realmatch.group(1) == ')':
if status == 'wait_string':
finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
errmsg = ("first argument of function '_ (...)' must be a "
"string")
wmlerr(finfo, errmsg)
else:
# if parenthesis is closed rightly after first argument
# this is not a plural form. So we set
# PendingLuaString.plural to None
if status == 'wait_plural':
pywmlx.state.machine._pending_luastring.plural = None
xline = xline[realmatch.end():]
return (xline, 'lua_idle')
else:
# this should never happen. But if the first value after _ (
# does not start with a string marker, than wmlxgettext will
# finish with an error
finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
errmsg = "first argument of function '_ (...)' must be a string"
wmlerr(finfo, errmsg)
xline = xline[realmatch.end():]
xline = realmatch.group(1) + xline
return (xline, _nextstate)
else:
# You can find a new line rigthly after _ (. In this case, we will
# continue to search a string into the next line
return (None, 'lua_plidle1')
def run(self, xline, lineno, match):
# if will ever happen 'wmlstr02 assertion error' than you could
# turn 'mydebug' to 'True' to inspect what exactly happened.
# However this should be never necessary
mydebug = False
_nextstate = 'wml_idle'
loc_translatable = True
loc_multiline = False
loc_string = None
# match group(1) exists, so it is a single line string
# (group(2) will not exist, than)
if match.group(1) is not None:
loc_multiline = False
loc_string = match.group(1)
xline = xline [ match.end(): ]
# match.group(2) exists, so it is a multi-line string
# (group(1) will not exist, than)
elif match.group(2) is not None:
loc_multiline = True
loc_string = match.group(2)
_nextstate = 'wml_str20'
xline = None
else:
if mydebug:
err_message = ("wmlstr02 assertion error (DEBUGGING)\n" +
'g1: ' + str(match.group(1)) + '\n' +
'g2: ' + str(match.group(2)) )
finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
wmlerr(finfo, err_message)
else:
wmlerr('wmlxgettext python sources',
'wmlstr02 assertion error\n'
'please report a bug if you encounter this error message')
pywmlx.state.machine._pending_wmlstring = (
pywmlx.state.machine.PendingWmlString(
lineno, loc_string, loc_multiline, loc_translatable
)
)
return (xline, _nextstate)
def run(*, filebuf, fileref, fileno, startstate, waitwml=True):
global _states
global _current_lineno
global _linenosub
global _waitwml
global _currentdomain
global _dictionary
global _pending_luafuncname
global _on_luatag
_pending_luafuncname = None
_on_luatag = False
# cs is "current state"
cs = _states.get(startstate)
cs_debug = startstate
_current_lineno = 0
_linenosub = 0
_waitwml = waitwml
_currentdomain = _initialdomain
pywmlx.nodemanip.newfile(fileref, fileno)
# debug_cs = startstate
try:
for xline in filebuf:
xline = xline.strip('\n\r')
_current_lineno += 1
# on new line, debug file will write another marker
if _debugmode:
print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@',
file=_fdebug)
while xline is not None:
# print debug infos (if debugmode is on)
if _debugmode:
lno = '%05d' % _current_lineno
print('---------------------------------------------------',
file=_fdebug)
print('LINE', lno, '|', xline, file=_fdebug)
# action number is used to know what function we should run
action = 0
v = None
m = None
if cs.regex is None:
# action = 1 --> execute state.run
action = 1
if _debugmode:
print('ALWAYS-RUN x', cs_debug, file=_fdebug)
else:
# m is match
m = re.match(cs.regex, xline)
if m:
# action = 1 --> execute state.run
action = 1
if _debugmode:
print('RUN state \\', cs_debug, file=_fdebug)
else:
# action = 2 --> change to the state pointed by
# state.iffail
action = 2
if _debugmode:
print('FAIL state |', cs_debug, file=_fdebug)
if action == 1:
# xline, ns: xline --> override xline with new value
# ns --> value of next state
xline, ns = cs.run(xline, _current_lineno, m)
cs_debug = ns
cs = _states.get(ns)
else:
cs_debug = cs.iffail
cs = _states.get(cs.iffail)
# end while xline
# end for xline
except UnicodeDecodeError as e:
if "test_cve_2018_1999023_2.cfg" in pywmlx.nodemanip.fileref:
# This unit test is allowed to contain invalid UTF-8. Ignore it.
return
errpos = int(e.start) # error position on file object with UTF-8 error
errbval = hex(e.object[errpos]) # value of byte wich causes UTF-8 error
# well... when exception occurred, the _current_lineno value
# was not updated at all due to the failure of the try block.
# (it is = 0)
# this means we need to make a workaround to obtain in what line of the
# file the problem happened.
# In order to perform this task (and not only) we create a temporary
# string wich contains all the file text UNTIL the UTF-8
untilerr_buf = e.object[0:errpos] # buffer containing file text
untilerr = "".join(map(chr, untilerr_buf))
# splituntil will be a array of strings (each item is a line of text).
# the last item will show the point where the invalid UTF-8 character
# was found.
splituntil = untilerr.split('\n')
# error line is equal of lines of text until error occurs (number of
# items on splituntil string array)
errlineno = len(splituntil)
# finally we can know the actual file info
finfo = pywmlx.nodemanip.fileref + ":" + str(errlineno)
errmsg = (
"UTF-8 Format error.\nCan't decode byte " + str(errbval) + ' (' +
e.reason + ').\n' +
'Probably your file is not encoded with UTF-8 encoding: you ' +
'should open the file with an advanced text editor, and re-save ' +
'it with UTF-8 encoding.\n' +
'To avoid this problem in the future, you might want to set ' +
'the default encoding of your editor to UTF-8.\n\n' +
'Text preceding the invalid byte (source file, line ' +
str(errlineno) + '):\n' + splituntil[-1] + '\n'
)
wmlerr(finfo, errmsg)
pywmlx.nodemanip.closefile(_dictionary, _current_lineno)
def run(*, filebuf, fileref, fileno, startstate, waitwml=True):
global _states
global _current_lineno
global _linenosub
global _waitwml
global _currentdomain
global _dictionary
global _pending_luafuncname
global _on_luatag
_pending_luafuncname = None
_on_luatag = False
# cs is "current state"
cs = _states.get(startstate)
cs_debug = startstate
_current_lineno = 0
_linenosub = 0
_waitwml = waitwml
_currentdomain = _initialdomain
pywmlx.nodemanip.newfile(fileref, fileno)
# debug_cs = startstate
try:
for xline in filebuf:
xline = xline.strip('\n\r')
_current_lineno += 1
# on new line, debug file will write another marker
if _debugmode:
print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@',
file=_fdebug)
while xline is not None:
# print debug infos (if debugmode is on)
if _debugmode:
lno = '%05d' % _current_lineno
print(
'---------------------------------------------------',
file=_fdebug)
print('LINE', lno, '|', xline, file=_fdebug)
# action number is used to know what function we should run
action = 0
v = None
m = None
if cs.regex is None:
# action = 1 --> execute state.run
action = 1
if _debugmode:
print('ALWAYS-RUN x', cs_debug, file=_fdebug)
else:
# m is match
m = re.match(cs.regex, xline)
if m:
# action = 1 --> execute state.run
action = 1
if _debugmode:
print('RUN state \\', cs_debug, file=_fdebug)
else:
# action = 2 --> change to the state pointed by
# state.iffail
action = 2
if _debugmode:
print('FAIL state |', cs_debug, file=_fdebug)
if action == 1:
# xline, ns: xline --> override xline with new value
# ns --> value of next state
xline, ns = cs.run(xline, _current_lineno, m)
cs_debug = ns
cs = _states.get(ns)
else:
cs_debug = cs.iffail
cs = _states.get(cs.iffail)
# end while xline
# end for xline
except UnicodeDecodeError as e:
errpos = int(e.start) # error position on file object with UTF-8 error
errbval = hex(
e.object[errpos]) # value of byte wich causes UTF-8 error
# well... when exception occurred, the _current_lineno value
# was not updated at all due to the failure of the try block.
# (it is = 0)
# this means we need to make a workaround to obtain in what line of the
# file the problem happened.
# In order to perform this task (and not only) we create a temporary
# string wich contains all the file text UNTIL the UTF-8
untilerr_buf = e.object[0:errpos] # buffer containing file text
untilerr = "".join(map(chr, untilerr_buf))
# splituntil will be a array of strings (each item is a line of text).
# the last item will show the point where the invalid UTF-8 character
# was found.
splituntil = untilerr.split('\n')
# error line is equal of lines of text until error occurs (number of
# items on splituntil string array)
errlineno = len(splituntil)
# finally we can know the actual file info
finfo = pywmlx.nodemanip.fileref + ":" + str(errlineno)
errmsg = (
"UTF-8 Format error.\nCan't decode byte " + str(errbval) + ' (' +
e.reason + ').\n' +
'Probably your file is not encoded with UTF-8 encoding: you ' +
'should open the file with an advanced text editor, and re-save ' +
'it with UTF-8 encoding.\n' +
'To avoid this problem in the future, you might want to set ' +
'the default encoding of your editor to UTF-8.\n\n' +
'Text preceding the invalid byte (source file, line ' +
str(errlineno) + '):\n' + splituntil[-1] + '\n')
wmlerr(finfo, errmsg)
pywmlx.nodemanip.closefile(_dictionary, _current_lineno)
