def get_tokens(code):
lexer = SnowLexer()
lexer.input(code, '')
tokens = [t for t in lexer]
tokens_with_code = [t for t in tokens if t.lexpos != -1]
return tokens, tokens_with_code
"""
Lexes a snow file given as first argument and returns its lexed tokens as json.
"""
import sys
import json
from lexer import SnowLexer
if __name__ == '__main__':
code = open(sys.argv[1]).read()
lexer = SnowLexer()
lexer.input(code, '')
tokens = map(lambda t: dict(type=t.type, value=t.value), list(lexer))
print json.dumps(tokens)
"""
Lexes a snow file given as first argument and returns its lexed tokens as json.
"""
import sys
import json
from lexer import SnowLexer
if __name__ == "__main__":
code = open(sys.argv[1]).read()
lexer = SnowLexer()
lexer.input(code, "")
tokens = map(lambda t: dict(type=t.type, value=t.value), list(lexer))
print json.dumps(tokens)
