def printTokens(path):
""" Print tokens in Hebrew Python module """
readline = codecs.open(path, 'rU', 'utf-8').readline
for (type, string, (srow, scol), (erow, ecol), line) \
in htokenize.generate_tokens(readline):
name = token.tok_name[type]
print '%s (%d, %d): "%s"' % (name, scol, ecol, string.encode('utf-8'))
def printTokens(path):
""" Print tokens in Hebrew Python module """
readline = codecs.open(path, 'rU', 'utf-8').readline
for (type, string, (srow, scol), (erow, ecol), line) \
in htokenize.generate_tokens(readline):
name = token.tok_name[type]
print '%s (%d, %d): "%s"' % (name, scol, ecol,
string.encode('utf-8'))
def translate(readline, func):
""" Translate HPython source to Python source """
result = StringIO()
position = 0
indent = ''
newline = True
for (type, string, (srow, scol), (erow, ecol), line) \
in htokenize.generate_tokens(readline):
# Add missing whitespace before tokens
result.write(u' ' * (scol - position))
position = ecol
# Handle indentation
if type == token.NEWLINE:
newline = True
result.write(string)
continue
elif type == token.INDENT:
newline = False
indent = string
result.write(indent)
continue
elif type == token.DEDENT:
indent = u' ' * ecol
continue
elif newline:
newline = False
result.write(indent)
# Handle other tokens
if type == token.NAME:
result.write(func(string))
else:
result.write(string)
return result.getvalue()
def translate(readline, func):
""" Translate HPython source to Python source """
result = StringIO()
position = 0
indent = ''
newline = True
for (type, string, (srow, scol), (erow, ecol), line) \
in htokenize.generate_tokens(readline):
# Add missing whitespace before tokens
result.write(u' ' * (scol - position))
position = ecol
# Handle indentation
if type == token.NEWLINE:
newline = True
result.write(string)
continue
elif type == token.INDENT:
newline = False
indent = string
result.write(indent)
continue
elif type == token.DEDENT:
indent = u' ' * ecol
continue
elif newline:
newline = False
result.write(indent)
# Handle other tokens
if type == token.NAME:
result.write(func(string))
else:
result.write(string)
return result.getvalue()
