def test_position():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
}
signed short d[123];
unsigned char e;
} StructName;
"""
lexer = Lexer(testcase)
iterator = iter(lexer)
next(iterator)
next(iterator)
next(iterator)
line = lexer.line_number
pos = lexer.line_pos
assert (line, pos) == (3, 22)
next(iterator)
next(iterator)
next(iterator)
line = lexer.line_number
pos = lexer.line_pos
assert (line, pos) == (4, 17)
next(iterator)
next(iterator)
line = lexer.line_number
pos = lexer.line_pos
assert (line, pos) == (5, 17)
def main():
text = ""
with open("src/sample-source-codes/test5.cfpl", "r") as file:
for line in file.readlines():
text += line.lstrip() + "\\n"
lexer = Lexer(text)
parser = Parser(lexer)
interpreter = Interpreter(parser)
interpreter.interpret()
def run(filename, text):
lexer = Lexer(text, filename)
tokens, error = lexer.make_token()
# make token segregate the input symbols
# according to tokens defined and return
# list of tokens and error (if any/None)
if error: return None, error
parser = Parser(tokens)
# Abstract syntax tree
ast = parser.parse()
if ast.error: return None, ast.error
# Run Program
interpreter = Interpreter()
context = Context('<progrom>')
context.symbolTable = globalSymbolTable
result = interpreter.visit(ast.node, context)
return result.value , result.error
def test_without_typedef():
testcase = """
//comment
struct FirstName {
int foo;
float bar[1];
} NameAlias;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure is not None
def test_multiple_names():
testcase = """
//comment
typedef struct StructName {
int foo;
float bar[MAX_SIZE];
} StructNameAlias;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure.name == 'StructNameAlias'
def main():
files = os.listdir(f"{ BASE_DIR }/src/sample-source-codes")
for file_name in files:
text = ""
with open(f"src/sample-source-codes/{file_name}", "r") as file:
for line in file.readlines():
text += line.lstrip() + "\\n"
lexer = Lexer(text)
parser = Parser(lexer)
interpreter = Interpreter(parser)
interpreter.interpret()
def test_with_error():
testcase = """
//comment
typedef struct {
int a;
struct {
double b;
} abc;
^#$Asd fssd;
} StructName;
"""
lexer = Lexer(testcase)
list(lexer)
def test_comment_before_bracket():
testcase = """
//comment
typedef struct // comment before opening bracket
{
int a;
unsigned long long int b;
/* comment
before
closing bracket */
} StructName;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure is not None
def test_all_tokens():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
}
signed short d[123];
unsigned char e[123 * (2 - 4) + 3 / 4 - a];
} StructName ;"""
lexer = Lexer(testcase)
lexemes = list(lexer)
for lexeme in lexemes:
print(lexeme)
assert len(lexemes) == 47
def test_correct_syntax():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
} inner_struct_name;
signed short d[123];
unsigned char e;
} StructName;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure.name == "StructName"
assert structure.description == "comment"
assert len(structure.variables) == 5
def test_unexpected_end_of_lexemes():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
} inner_struct_name[5];
signed short d[123];
unsigned char e;
}
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
try:
parser.parse()
except ParserError as err:
assert err.message.endswith('end of lexemes found')
else:
assert False
def test_wrong_lexeme_error():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long struct c;
} inner_struct_name;
signed short d[123];
unsigned char e;
} StructName;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
try:
parser.parse()
except ParserError as err:
assert err.message.startswith('Wrong')
else:
assert False
def test_end_expected_but_lexeme_found():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
} inner_struct_name;
signed short d[123];
unsigned char e;
} StructName; int a = 5;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
try:
parser.parse()
except ParserError as err:
assert err.message.startswith('End of lexemes')
else:
assert False
import argparse
from lib.stoneToken import StoneToken
from lib.lexer import LineReader
from lib.lexer import Lexer
if __name__ == "__main__":
# use --file or -f set filename, default is samples/first.stone
argparser = argparse.ArgumentParser()
argparser.add_argument("--file", "-f", type=str)
args = argparser.parse_args()
filename = "samples/first.stone"
if args.file is not None:
filename = args.file
print("File: " + filename)
with open(filename, 'r') as f:
reader = LineReader(f)
lexer = Lexer(reader)
token = lexer.read()
if token != None: print(token.getText())
while token != StoneToken.EOF:
token = lexer.read()
if token != None: print(token.getText())
# -*- coding: utf-8 -*-
import sys, traceback
from lib.lexer import Lexer
from lib.syntactic import Syntactic
fname = 'sources/example1.pas'
if len(sys.argv) > 1:
fname = sys.argv[1]
lexer = Lexer(fname)
try:
tokens = lexer.parse()
except Exception as e:
print e
sys.exit(1)
template = "{0:10} {1:25} {2:5}" # column widths: 10, 20, 5
# print template.format('Token', 'Classificacao', 'Linha')
# for tok in tokens:
# print template.format(*tok.get_tuple())
syntactic = Syntactic(tokens)
try:
if syntactic.parse() == False:
print "Erro sintático"
except Exception as e:
def __init__(self):
self.lexer = Lexer()
self.tokens = self.lexer.tokens
# TODO: debug=False & write_tables=False when ready
self.parser = yacc.yacc(module=self,
debug=False)
import os, sys, time, re
from lib import application
from lib.lexer import Lexer
scan_sources = [
("title", "titles", "object_id", "data", 3),
("text", "text", "object_id", "data", 2),
("nick", "users", "object_id", "nick", 2),
("type", "objects", "id", "type", 1),
]
environ = { "REMOTE_ADDR" : "localhost" }
app = application.Application( environ, lambda x,y: None, path=os.path.dirname(sys.argv[0]) )
c = app.db.cursor()
for scan_source_definition in scan_sources:
scan_source, scan_table, id_column, data_column, rank = scan_source_definition
scan_time = int( time.time() )
#c.execute( """PRAGMA synchronous = 0;""" )
rows = c.execute( """select %(id_column)s, %(data_column)s from %(scan_table)s""" % locals() ).fetchall()
for obj_id, data in rows:
words = Lexer.scan( data )
c.execute( """delete from keywords where object_id=? and scan_source=?""", [obj_id, scan_source] )
for pos, word in words:
c.execute( """insert into keywords (object_id, word, pos, rank, scan_source, scan_time)
values(?, ?, ?, ?, ?, ?)""", [obj_id, word, pos, rank, scan_source, scan_time] )
app.close_db( commit=True ) # also commits pending changes
def __init__(self):
self.lexer = Lexer()
self.tokens = self.lexer.tokens
self.parser = yacc.yacc(module=self, write_tables=0, debug=False)