def buildMatrix(fname, rows, cols):
# Construct an empty matrix with specified rows and cols
matrix = []
for i in range(0, rows):
matrixCols = []
for i in range(0, cols):
matrixCols.append(0)
matrix.append(matrixCols)
# Open the file using Scanner or File Pointer
# scanner example from http://troll.cs.ua.edu/cs150/projects/practice1.html
if (fname):
s = Scanner(fname)
# Read the data from each row into an array
scannedArray = []
token = s.readtoken()
while token != '':
scannedArray.append(token)
token = s.readtoken()
# Append the array onto the matrix
for rowNumber in range(0, len(matrix)):
for columnNumber in range(0, len(matrix[rowNumber])):
if (len(scannedArray) > 0):
matrix[rowNumber][columnNumber] = scannedArray.pop(0)
# Close the file
s.close()
return matrix
def folder_scan(self, folders):
class TimedProgressDisplay:
def __init__(self, name, interval = 5):
self.__name = name
self.__interval = interval
self.__last_display = 0
def __call__(self, scanned, total):
if time.time() - self.__last_display > self.__interval or scanned == total:
print "Scanning '{0}': {1}% ({2}/{3})".format(self.__name, (scanned * 100) / total, scanned, total)
self.__last_display = time.time()
s = Scanner(db.session)
if folders:
folders = map(lambda n: db.Folder.query.filter(db.Folder.name == n and db.Folder.root == True).first() or n, folders)
if any(map(lambda f: isinstance(f, basestring), folders)):
print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring))
for folder in filter(lambda f: isinstance(f, db.Folder), folders):
FolderManager.scan(folder.id, s, TimedProgressDisplay(folder.name))
else:
for folder in db.Folder.query.filter(db.Folder.root == True):
FolderManager.scan(folder.id, s, TimedProgressDisplay(folder.name))
added, deleted = s.stats()
db.session.commit()
print "Scanning done"
print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2])
print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2])
def main():
args_handler = Handler()
ip = args_handler.parsing_ip()
tgt.send('1.1.1.1')
scan = Scanner(tgt)
scan.scan_ports()
class Parser:
def __init__(self, text=''):
self.lex = Scanner(text) # make a scanner
self.vars = {'pi':3.14159} # add constants
self.traceme = TraceDefault
def parse(self, *text): # external interface
if text:
self.lex.newtext(text[0]) # reuse with new text
tree = self.analyse() # parse string
if tree:
if self.traceme: # dump parse-tree?
print; tree.trace(0)
if self.errorCheck(tree): # check names
self.interpret(tree) # evaluate tree
def analyse(self):
try:
self.lex.scan() # get first token
return self.Goal() # build a parse-tree
except SyntaxError:
print 'Syntax Error at column:', self.lex.start
self.lex.showerror()
except LexicalError:
print 'Lexical Error at column:', self.lex.start
self.lex.showerror()
def errorCheck(self, tree):
try:
tree.validate(self.vars) # error checker
return 'ok'
except UndefinedError, varinfo:
print "'%s' is undefined at column: %d" % varinfo
self.lex.start = varinfo[1]
self.lex.showerror() # returns None
def test_can_scan(self):
self.skipTest('Don\'t know how to test with SSH :/')
scanner = Scanner(network_tools=NetworkToolsStub(), hostname='10.0.1.231')
scan = scanner.scan(port=9100)
self.assertDictEqual(scan, {})
def run(sniffer_instance=None, wait_time=0.5, clear=True, args=(), debug=False):
"""
Runs the auto tester loop. Internally, the runner instanciates the sniffer_cls and
scanner class.
``sniffer_instance`` The class to run. Usually this is set to but a subclass of scanner.
Defaults to Sniffer. Sniffer class documentation for more information.
``wait_time`` The time, in seconds, to wait between polls. This is dependent on
the underlying scanner implementation. OS-specific libraries may choose
to ignore this parameter. Defaults to 0.5 seconds.
``clear`` Boolean. Set to True to clear the terminal before running the sniffer,
(alias, the unit tests). Defaults to True.
``args`` The arguments to pass to the sniffer/test runner. Defaults to ().
``debug`` Boolean. Sets the scanner and sniffer in debug mode, printing more internal
information. Defaults to False (and should usually be False).
"""
if sniffer_instance is None:
sniffer_instance = ScentSniffer()
if debug:
scanner = Scanner(('.',), logger=sys.stdout)
else:
scanner = Scanner(('.',))
#sniffer = sniffer_cls(tuple(args), clear, debug)
sniffer_instance.set_up(tuple(args), clear, debug)
sniffer_instance.observe_scanner(scanner)
scanner.loop(wait_time)
def compile_pascal(source, dest, is_debug = False, is_interpret = False, out_stream = sys.stdout, output_tokens = False, output_bytecodes = False, lib = ['.'], in_stream = sys.stdin):
'''
DID YOU KNOW that compile() is a built in function?
'''
set_debug(is_debug)
debug("Compiling %s into %s" % (source, dest))
scanner = Scanner(source)
tokens = scanner.scan()
if output_tokens:
write(tokens, source + "_tokenized")
debug('scanning complete')
parser = Parser(tokens, source, lib = lib)
bytecodes, success = parser.parse()
if output_bytecodes:
if is_debug:
write(prettify(bytecodes), source + "_unassembled")
else:
write(bytecodes, source + "_unassembled")
if not success:
print 'Parsing error'
return
debug('parsing complete')
assembler = Assembler(bytecodes)
assembled = assembler.assemble()
if is_debug:
write(prettify(assembled), dest + '_debug')
write(assembled, dest)
debug('assembly complete.' )
if is_interpret:
interp = Interpreter(out_stream, in_stream, code = assembled)
interp.interpret()
else:
debug('run program now with `python interpreter.py %s`' % dest)
def main():
'''
sets the structure for going through the dream process step by step
'''
fullname = get_name(kind='in') # read in filename
outname = get_name(kind='out') # read in output name
opts = get_options() # read in options
# start dream computations
s = Scanner() # new scanner
s.read_alpha_png(fullname) # read and store location of transparent pixels
my_image = Image.open(fullname, 'r') # open image
d = Dreamer(my_image) # create new dreamer with reference to image object
img = iterate(d, my_image, outname, opts, save=False) # this function is for iterating over the same image multiple times
out = s.cut_jpg(img) # send dream buffer object to be cut into transparent PNG
new_image = Image.new('RGBA', my_image.size) # new blank image of same dimensions
new_image.putdata(out) # make new image from the information from s.cut_jpg
new_image.save(outname) # export image
print("\nsaved image as: %s" % outname)
open_image(outname) # give user option to automatically open image
def test_multipleinstances(self):
report = Report.from_string(self.MULTIPLEINSTANCES)
scanner = Scanner(report, [MultipleInstances()])
for ev in scanner.parse():
report.add_event(ev)
self.logger.log(logging.INFO, "event: %s", str(ev))
self.assertEqual(report.count(), 1)
report.report()
def test_build_fail_two(self):
report = Report.from_string(self.BUILD_FAIL_2)
scanner = Scanner(report, [BuildFail()])
for ev in scanner.parse():
report.add_event(ev)
self.logger.log(logging.DEBUG, "event: %s", str(ev))
self.assertEqual(report.count(), 2)
report.report()
def test_skipped_conflict(self):
report = Report.from_string(self.SKIPPED_CONFLICT)
scanner = Scanner(report, [SkippedConflict()])
for ev in scanner.parse():
report.add_event(ev)
self.logger.log(logging.DEBUG, "event: %s", str(ev))
report.report()
self.assertEqual(report.count(), 1)
class Tray(QtWidgets.QSystemTrayIcon):
def __init__(self, client, icon, parent=None):
QtWidgets.QSystemTrayIcon.__init__(self, icon, parent)
self.client = client
self.icon = icon
self.user = self.client.get_account('me').url
self.scanner = Scanner(self.client, self)
self.options = OptionsWindow(self.client, self.scanner, self)
self.options.init_options()
self.stop_event = threading.Event()
self.scan_thread = threading.Thread(target=self.scanner.scan, args=(self.stop_event,))
self.scan_thread.start()
menu = QtWidgets.QMenu(parent)
exitAction = QtWidgets.QAction("&Quit ", self)
exitAction.setShortcut("Ctrl+Q")
exitAction.setStatusTip('Good bye')
exitAction.triggered.connect(self.appExit)
optAction = QtWidgets.QAction("&Options... ", self)
optAction.setShortcut("Ctrl+O")
optAction.setStatusTip("Customize")
optAction.triggered.connect(self.show_options)
sendAction = QtWidgets.QAction("Copy Link of Last Uploaded Image", self)
sendAction.setShortcut("Ctrl+S")
sendAction.setStatusTip("...")
sendAction.triggered.connect(self.copy_last)
menu.addAction(sendAction)
menu.addAction(optAction)
menu.addSeparator()
menu.addAction(exitAction)
self.setContextMenu(menu)
def appExit(self):
# Reset OSX Screenshot storage
with open(os.devnull, 'w') as nul:
path = os.path.expanduser('~') + '/Desktop/'
subprocess.call(["defaults", "write", "com.apple.screencapture", "location", path])
subprocess.call(["killAll", "SystemUIServer"])
kill_proc_tree(me)
self.stop_event.set()
self.scanner.scan(self.stop_event)
sys.exit()
def show_options(self):
self.options.show_opts()
def copy_last(self):
if self.scanner.loader.link == '':
self.scanner.loader.to_clipboard()
else:
self.scanner.loader.to_clipboard()
self.scanner.loader.copy_notification()
def readTable(filename):
s = Scanner(filename)
table = []
record = readRecord(s)
while record != "":
table.append(record)
record = readRecord(s)
s.close()
return table
def mainScanner():
file = open(filer)
scanner = Scanner(file.read() + '\0')
lexeme = scanner.scanNext()
while lexeme.type != 200:
lexeme.pprint()
lexeme = scanner.scanNext()
def run(self):
parsedArgs = self.options.parse()
scanner = Scanner(parsedArgs.path)
duplicateCollector = DuplicateCollector()
scanner.scan(duplicateCollector)
if parsedArgs.verbose:
duplicateCollector.write(True)
else:
duplicateCollector.write()
def test_3():
s = Scanner()
input_string = 'naqsh faryaadii hai kis kii sho;xii-e ta;hriir kaa'
scan_results = s.scan(input_string, known_only=True)
assert len(scan_results['results']) == 1
result = scan_results['results'][0]
assert len(scan_results['results']) == 1
assert result['scan'] == '=-===-===-===-=', result['scan']
class MainController:
def __init__(self, app):
self.deviceData = DeviceData()
self.view = MainView(None)
self.view.scanForDevices.Bind(wx.EVT_BUTTON, self.ScanForDevices)
self.view.syncActivities.Bind(wx.EVT_BUTTON, self.SyncActivities)
self.view.Bind(wx.EVT_MENU, self.OnAbout, self.view.aboutMenuItem)
self.view.Bind(wx.EVT_MENU, self.OnExit, self.view.exitMenuItem)
self.view.Show()
self.scanner = Scanner()
## TODO Preferences for Selected Scanners
self.scanner.addScanner(AntScanner())
self.connector = Connector()
## TODO Preferences for Selected Connectors
self.connector.addConnector(GarminConnector())
pub.subscribe(self.ScanningStarted, "SCANNING STARTED")
pub.subscribe(self.DeviceDetected, "DEVICE DETECTED")
pub.subscribe(self.ActivityRetrieved, "ACTIVITY RETRIEVED")
pub.subscribe(self.ScanningEnded, "SCANNING ENDED")
pub.subscribe(self.SyncStarted, "SYNC STARTED")
pub.subscribe(self.SyncEnded, "SYNC ENDED")
pub.subscribe(self.LoginSuccesful, "LOGIN SUCCESFUL")
pub.subscribe(self.LoginFailed, "LOGIN FAILED")
pub.subscribe(self.ActivitiesUploaded, "ACTIVITIES UPLOADED")
def ScanForDevices(self, evt):
self.scanner.scan()
def ScanningStarted(self, evt):
self.view.setStatus("Scanning started")
def ScanningEnded(self, evt):
self.view.setStatus("Scanning ended")
def DeviceDetected(self, evt):
self.view.setStatus("Device detected")
def ActivityRetrieved(self, evt):
self.view.setStatus("Retrieved activity")
def SyncActivities(self, evt):
self.connector.sync()
def SyncStarted(self, evt):
self.view.setStatus("Sync started")
def SyncEnded(self, evt):
self.view.setStatus("Sync ended")
def LoginSuccesful(self, evt):
self.view.setStatus("Login Succesful")
def LoginFailed(self, evt):
self.view.setStatus("Login Failed")
def ActivitiesUploaded(self, evt):
self.view.setStatus("Activities Uploaded")
def OnExit(self,e):
self.Close(True)
def OnAbout(self, event):
dlg = wx.MessageDialog( self.view, "A community-developed Linux version of the ANT Agent. Supports Garmin-based fitness devices that communicate either over USB serial or via the ANT USB connector. Developed by Philip Whitehouse, based on work by Braiden Kindt, Gustav Tiger and Collin (cpfair). Copyright 2014", "About ANT Agent for Linux", wx.OK);
dlg.ShowModal()
dlg.Destroy()
def test_01():
text_XX = """
@@ aa
"""
def callback(mo):
D("callback %r", mo)
scanner = Scanner((
(r'^@@', callback),
(r'aaa', callback),
), re.MULTILINE)
scanner.scan(text_XX)
def run_parser(self, file_, lib_classes = None):
"""parses a jml file and returns a list of abstract syntax trees, or
parses a string containing JaML code.
"""
scanner = Scanner()
# Try and parse a file, if this fails, parse as a string
try:
# This is to see if it's a file or not
open(file_)
# Stores the directory of the main class
dir_ = os.path.dirname(file_)
# Add the name of the main class to begin with
file_name = os.path.basename(file_)
class_name = file_name.replace('.jml', '')
# this stores names of all already seen class references
seen = [class_name]
# Stores classes to parse
to_parse = [class_name]
# Stores the ASTs of parsed classes
parsed = nodes.ProgramASTs()
while to_parse:
# Append file information to the class names
file_name = os.path.join(dir_, to_parse[0] + '.jml')
# Get the raw input
raw = open(file_name)
input_ = raw.read()
# Scan and parse the file
tokens = scanner.tokenize(input_)
ast = self.parse(tokens)
# Check the class and file are named the same
if to_parse[0] != ast.children[0].value:
msg = 'Class name and file name do not match!'
raise NameError(msg)
to_parse.pop(0)
parsed.append(ast)
# Fined classes reference from the one just parsed
if lib_classes == None:
lib_classes = {}
refed_classes = self._find_refed_classes
refed_classes = refed_classes(ast, seen, [], lib_classes, dir_)
seen += refed_classes
to_parse += refed_classes
return parsed
except IOError:
# Simply parse the program held in the string
tokens = scanner.tokenize(file_)
ast = self.parse(tokens)
ast_wrapper = nodes.ProgramASTs()
ast_wrapper.append(ast)
return ast_wrapper
def main(argv):
if len(argv) == 1 or len(argv) > 2:
usage()
sys.exit(2)
try:
path = argv[1]
scanner = Scanner()
scanner.scan(path)
scanner.rebuild_tracks()
fcpxml = FcpXML()
fcpxml.create_xml()
except getopt.GetoptError, err:
print str(err)
usage()
sys.exit(2)
def main():
# Include current path in case we attempt to load modules.
sys.path.append(os.getcwd())
parser = argparse.ArgumentParser("Runs a set of tests against the set of provided URLs")
parser.add_argument("-u", "--url", action="append", dest="targets", help="Add a target to test")
parser.add_argument("-f", "--target-file", action="append", dest="target_files", help="File with URLs to test")
parser.add_argument("-m", "--module", action="append", default = [], dest="modules", help="Load an extension module")
parser.add_argument("-D", "--disable-core", action="store_true", default = False, dest="disable_core", help="Disable corechecks")
parser.add_argument("-p", "--force-passive", action="store_true", default=False, dest="force_passives", help ="Force passives to be run for each active test")
parser.add_argument("-d", "--dns", action="store_false", default=True, dest="resolve_target", help ="Skip DNS resolution when registering a target")
parser.add_argument("-r", "--report", action="store", default="xml", dest="report",help="Load a reporter e.g. -r reporter.AntXmlReporter")
parser.add_argument("-o", "--output", action="store", default="garmr-results.xml", dest="output", help="Default output is garmr-results.xml")
parser.add_argument("-c", "--check", action="append", dest="opts", help="Set a parameter for a check (check:opt=value)" )
parser.add_argument("-e", "--exclude", action="append", dest="exclusions", help="Prevent a check from being run/processed")
parser.add_argument("--save", action="store", dest="dump_path", help="Write out a configuration file based on parameters (won't run scan)")
args = parser.parse_args()
scanner = Scanner()
scanner.force_passives = args.force_passives
scanner.resolve_target = args.resolve_target
scanner.output = args.output
# Start building target list.
if args.targets != None:
for target in args.targets:
scanner.register_target(target)
# Add targets from files to the list.
if args.target_files != None:
for targets in args.target_files:
try:
f = open(targets, "r")
for target in f:
t = target.strip()
if len(t) > 0:
scanner.register_target(t)
except:
Scanner.logger.error("Unable to process the target list in: %s", targets)
# Load built-in modules if required.
if args.disable_core == False:
corechecks.configure(scanner)
# Configure modules.
# TODO: change the module loading to scan the list of classes in a module and automagically
# detect any tests defined.
if args.modules != None:
for module in args.modules:
try:
__import__(module)
m = sys.modules[module]
m.configure(scanner)
except Exception, e:
Scanner.logger.fatal("Unable to load the requested module [%s]: %s", module, e)
quit()
def parse(self, sourceStr):
self._completionMessage = "No errors"
self._parseSuccessful = True
self._scanner = Scanner(sourceStr)
self._expression()
self._accept(self._scanner.get(), Token.EOE,
"symbol after end of expression")
def folder_prune():
s = Scanner(session)
folders = session.query(Folder).filter(Folder.root == True)
if folders:
for folder in folders:
print "Pruning: " + folder.path
FolderManager.prune(folder.id, s)
added, deleted = s.stats()
print "\a"
print "Pruning done"
print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2])
print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2])
def __init__(self, client, icon, parent=None):
QtWidgets.QSystemTrayIcon.__init__(self, icon, parent)
self.client = client
self.icon = icon
self.user = self.client.get_account('me').url
self.scanner = Scanner(self.client, self)
self.options = OptionsWindow(self.client, self.scanner, self)
self.options.init_options()
self.stop_event = threading.Event()
self.scan_thread = threading.Thread(target=self.scanner.scan, args=(self.stop_event,))
self.scan_thread.start()
menu = QtWidgets.QMenu(parent)
exitAction = QtWidgets.QAction("&Quit ", self)
exitAction.setShortcut("Ctrl+Q")
exitAction.setStatusTip('Good bye')
exitAction.triggered.connect(self.appExit)
optAction = QtWidgets.QAction("&Options... ", self)
optAction.setShortcut("Ctrl+O")
optAction.setStatusTip("Customize")
optAction.triggered.connect(self.show_options)
sendAction = QtWidgets.QAction("Copy Link of Last Uploaded Image", self)
sendAction.setShortcut("Ctrl+S")
sendAction.setStatusTip("...")
sendAction.triggered.connect(self.copy_last)
menu.addAction(sendAction)
menu.addAction(optAction)
menu.addSeparator()
menu.addAction(exitAction)
self.setContextMenu(menu)
def createSearchArray(size):
command = str.format("python3 makeintegers.py {0} 0 1 {1} > data.out", size, swaps)
# create a file called data.out with the randomized array
subprocess.call(command, shell=True)
numFile = Scanner("data.out")
searchArray = DynamicArray(size)
# create a DynamicArray with the randomized search data
while True:
token = numFile.readint()
if token == "":
break
else:
searchArray.addToBack(token)
return searchArray
def run(cls):
s3 = boto.connect_s3()
prod_bucket = s3.get_bucket('dray-assets-prod')
load_files = list(prod_bucket.list(prefix="media/loads"))
document_files = [f for f in load_files if "load_action_approvals" not in f.name]
for f in document_files:
# Check for existing scanned version
filename, file_extension = os.path.splitext(f.key)
result_keyname = filename + "-scanned" + file_extension
possible_key = prod_bucket.get_key(result_keyname)
if possible_key is not None or "-scanned" in f.key:
# Scanned file already exists, move on
continue
# Read image and scan
image = prod_bucket.get_key(f.key)
image.get_contents_to_filename('tmp/tmp-src.png')
result_filename = Scanner.scan('tmp/tmp-src.png')
# Create new key for result file
k = Key(prod_bucket)
k.key = result_keyname
k.set_contents_from_filename(result_filename)
def scan_folder(id = None):
s = Scanner(session)
if id is None:
for folder in Folder.query.filter(Folder.root == True):
FolderManager.scan(folder.id, s)
else:
status = FolderManager.scan(id, s)
if status != FolderManager.SUCCESS:
flash(FolderManager.error_str(status))
return redirect(url_for('folder_index'))
added, deleted = s.stats()
session.commit()
flash('Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]))
flash('Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2]))
return redirect(url_for('folder_index'))
def __init__(self, settings):
"""TODO: to be defined1. """
self.cubo = Cubo()
self.cuboescaneado = CuboEscaneado()
self.settings = settings
self.scanner = Scanner(self.cubo, self.settings)
self.robot = Robot()
self.solver = Solver(self.cubo)
def handle(self):
from scanner import Scanner
s = Scanner()
s.check_directories()
s.initialize_logger()
s.check_yara_file()
# Check in case client terminates connection mid xfer
self.request.settimeout(10.0)
try:
raw_msg_len = self.request.recv(4)
msg_len = struct.unpack('>I', raw_msg_len)[0]
data = ''
# Basic client request integrity check, not fullproof
proto_check = self.request.recv(7)
if proto_check != 'FSF_RPC':
s.dbg_h.error('%s Client request integrity check failed. Invalid FSF protocol used.' % dt.now())
raise ValueError()
while len(data) < msg_len:
recv_buff = self.request.recv(msg_len - len(data))
data += recv_buff
self.request.settimeout(None)
self.process_data(data, s)
except:
e = sys.exc_info()[0]
s.dbg_h.error('%s There was a problem processing the connection request from %s. Error: %s' % (dt.now(), self.request.getpeername()[0], e))
finally:
self.request.close()
def handle(self):
from scanner import Scanner
s = Scanner()
s.check_directories()
s.initialize_logger()
s.check_yara_file()
try:
raw_msg_len = self.request.recv(4)
msg_len = struct.unpack('>I', raw_msg_len)[0]
data = ''
while len(data) < msg_len:
recv_buff = self.request.recv(msg_len - len(data))
data += recv_buff
self.process_data(data, s)
except:
e = sys.exc_info()[0]
s.dbg_h.error('%s There was a problem processing the connection request from %s. Error: %s' % (dt.now(), self.request.getpeername()[0], e))
finally:
self.request.close()
or stack.peek().getPrecedence() <
token.getPrecedence()):
break
self._pfTokens.append(stack.pop())
stack.push(token)
for i in range(len(stack)):
self._pfTokens.append(stack.pop())
return self._pfTokens
def __inter__(self):
if not self._pfTokens:
self.convert()
return iter(self._pfTokens)
def __str__(self):
if not self._pfTokens:
self.convert()
return ' '.join(map(str, self._pfTokens))
if __name__ == '__main__':
ife = '(4+5)*(6-3)'
scanner = Scanner(ife)
print('Infix expression:', scanner)
converter = IFToPFConverter(scanner)
tokenList = converter.convert()
print(*tokenList)
print(converter)
args = sys.argv
if len(args) > 2:
print("Usage: ./lox [script]")
sys.exit(64)
from scanner import Scanner
from parser import Parser, ParseError
from interpreter import Interpreter
from resolver import Resolver
if len(args) == 2:
filename = args[1]
with open(filename, "r") as f:
data = f.read()
tokens = Scanner(data).scan_tokens()
statements = Parser(tokens).parse()
if not Lox.had_error:
interpreter = Interpreter()
Resolver(interpreter).resolve(statements)
if not Lox.had_error:
interpreter.interpret(statements)
else:
print("Lox 0.1.0")
interpreter = Interpreter()
try:
while True:
line = input("> ")
tokens = Scanner(line).scan_tokens()
statements = Parser(tokens).parse()
if not Lox.had_error:
class Mparser(object):
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
self.args = {}
self.const = {}
tokens = Scanner.tokens
precedence = (
("nonassoc", 'IFX'),
("nonassoc", 'ELSE'),
("nonassoc", 'LT', 'GT', 'EQ', 'NE', 'LE', 'GE'),
("left", '+', '-'),
("left", '*', '/'),
("left", 'DOTADD', 'DOTSUB'),
("left", 'DOTMUL', 'DOTDIV'),
)
def p_error(self, p):
if p:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".
format(p.lineno, self.scanner.find_tok_column(p), p.type,
p.value))
else:
print('At end of input')
def p_program(self, p):
"""program : instructions"""
p[0] = AST.Program(p[1])
print(p[0])
def p_instructions(self, p):
"""instructions : instructions instruction
| instruction """
if len(p) == 3:
p[1].instrs.append(p[2])
p[0] = p[1]
else:
p[0] = AST.Instructions(p[1])
def p_instruction(self, p):
"""instruction : print_instr
| assignment
| choice_instr
| while_instr
| for_instr
| return_instr
| break_instr
| continue_instr
| compound_instr"""
p[0] = p[1]
def p_print_instr(self, p):
"""print_instr : PRINT expr_list ';'
| PRINT error ';' """
p[0] = AST.Print(p[2])
def p_assignment(self, p):
"""assignment : id assign_operator expression ';'
| ref assign_operator expression ';' """
p[0] = AST.Assignment(p[1], p[2], p[3])
def p_ref(self, p):
"""ref : id '[' expr_list ']' """
p[0] = AST.Ref(p[1], p[3])
def p_assign_operator(self, p):
"""assign_operator : '='
| ADDASSIGN
| SUBASSIGN
| MULASSIGN
| DIVASSIGN"""
p[0] = AST.Assign_operator(p[1])
def p_vector(self, p):
"""vector : '[' expressions ']' """
p[0] = AST.Vector(p[2])
def p_expressions_or_vectors(self, p):
"""expressions : expression
| expressions ',' expression """
if len(p) == 4:
p[1].exprs.append(p[3])
p[0] = p[1]
else:
p[0] = AST.Expressions(p[1])
def p_choice_instr(self, p):
"""choice_instr : IF '(' condition ')' instruction %prec IFX
| IF '(' condition ')' instruction ELSE instruction
| IF '(' error ')' instruction %prec IFX
| IF '(' error ')' instruction ELSE instruction """
if len(p) == 6:
p[0] = AST.Choice(p[3], p[5])
else:
p[0] = AST.Choice(p[3], p[5], p[7])
def p_while_instr(self, p):
"""while_instr : WHILE '(' condition ')' instruction
| WHILE '(' error ')' instruction """
p[0] = AST.While(p[3], p[5])
def p_for_instr(self, p):
"""for_instr : FOR id '=' range instruction"""
p[0] = AST.For(p[2], p[4], p[5])
def p_range(self, p):
"""range : expression ':' expression"""
p[0] = AST.Range(p[1], p[3])
def p_return_instr(self, p):
"""return_instr : RETURN expression ';' """
p[0] = AST.Return(p[2])
def p_continue_instr(self, p):
"""continue_instr : CONTINUE ';' """
p[0] = AST.Continue()
def p_break_instr(self, p):
"""break_instr : BREAK ';' """
p[0] = AST.Break()
def p_compound_instr(self, p):
"""compound_instr : '{' instructions '}' """
p[0] = AST.ComInstructions(p[2])
def p_condition(self, p):
"""condition : expression"""
p[0] = p[1]
def p_const(self, p):
"""const : INTNUM
| FLOATNUM
| STRING"""
p[0] = AST.Const(p[1])
def p_expression(self, p):
"""expression : const
| id
| ref
| vector
| matrix_operation
| matrix_function
| minus_matrix
| matrix_transposed
| expression '+' expression
| expression '-' expression
| expression '*' expression
| expression '/' expression
| expression EQ expression
| expression NE expression
| expression LT expression
| expression GT expression
| expression LE expression
| expression GE expression
| '(' expression ')'
| '(' error ')'"""
if len(p) == 2:
p[0] = p[1]
elif len(p) == 4:
if p[1] == '(':
p[0] = p[2]
else:
if p[2] in ['+', '-', '*', '/']:
p[0] = AST.BinExpr(p[1], p[2], p[3])
else:
p[0] = AST.Condition(p[1], p[2], p[3])
def p_id(self, p):
"""id : ID"""
p[0] = AST.Variable(p[1], 0)
def p_matrix_operation(self, p):
"""matrix_operation : matrix dot_operation matrix"""
p[0] = AST.Matrix_operation(p[1], p[2], p[3])
def p_dot_operation(self, p):
"""dot_operation : DOTADD
| DOTSUB
| DOTMUL
| DOTDIV"""
p[0] = AST.Dot_operation(p[1])
def p_matrix(self, p):
"""matrix : id
| minus_matrix
| matrix_transposed"""
p[0] = AST.Matrix(p[1])
def p_matrix_transposed(self, p):
"""matrix_transposed : matrix "'" """
p[0] = AST.Matrix_transposed(p[1])
def p_minus_matrix(self, p):
"""minus_matrix : "-" matrix """
p[0] = AST.Minus_matrix(p[2])
def p_matrix_function(self, p):
"""matrix_function : ZEROS '(' expressions ')'
| ONES '(' expressions ')'
| EYE '(' expressions ')' """
p[0] = AST.Matrix_function(p[1], p[3])
def p_expr_list(self, p):
"""expr_list : expr_list ',' expression
| expression """
if len(p) == 4:
p[1].exprs.append(p[3])
p[0] = p[1]
else:
p[0] = AST.Expressions(p[1])
self.look_ahead].startswith('boollit') or self.tokens[
self.look_ahead] == 'LP':
term.add_child(self.factor())
if self.tokens[self.look_ahead].startswith('MULTIPLICATIVE'):
term.add_child(self.match('MULTIPLICATIVE'))
term.add_child(self.term())
return term
def factor(self):
factor = Node('<factor>')
if self.tokens[self.look_ahead].startswith('ident'):
factor.add_child(self.match('ident'))
elif self.tokens[self.look_ahead].startswith('num'):
factor.add_child(self.match('num'))
elif self.tokens[self.look_ahead].startswith('boollit'):
factor.add_child(self.match('boollit'))
elif self.tokens[self.look_ahead] == 'LP':
factor.add_child(self.match('LP'))
factor.add_child(self.simple_expression())
factor.add_child(self.match('RP'))
return factor
if __name__ == '__main__':
scanner = Scanner()
parser = Parser(scanner)
parser.parse()
from checker import Checker
from scanner import Scanner
from parser import Parser
from encoder import Encoder
fileDir = './example_files/prog1.txt'
if __name__ == "__main__":
scanner = Scanner(fileDir)
parser = Parser(scanner)
checker = Checker()
encoder = Encoder()
program = parser.parse_program()
checker.check(program)
encoder.encode(program)
encoder.save_target_program('examples_files/prog1.tam')
# print(json.dumps(program, default=lambda x: x.__dict__, indent=2))
class Cparser(object):
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
tokens = Scanner.tokens
precedence = (
("nonassoc", 'IFX'),
("nonassoc", 'ELSE'),
("right", '='),
("left", 'OR'),
("left", 'AND'),
("left", '|'),
("left", '^'),
("left", '&'),
("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'),
("left", 'SHL', 'SHR'),
("left", '+', '-'),
("left", '*', '/', '%'),
)
def p_error(self, p):
if p:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
else:
print('At end of input')
def p_program(self, p):
"""program : declarations fundefs instructions"""
p[0] = AST.Program(p.lineno(1), p[1], p[2], p[3])
print(p[0])
def p_declarations(self, p):
"""declarations : declarations declaration
| """
if len(p) == 3:
p[0] = AST.Declarations(p.lineno(1), p[1], p[2])
else:
p[0] = AST.Epsilon(p.lineno(0))
def p_declaration(self, p):
"""declaration : TYPE inits ';'
| error ';' """
if len(p) == 4:
p[0] = AST.Declaration(p.lineno(1), p[2], p[1])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
def p_inits(self, p):
"""inits : inits ',' init
| init """
if len(p) == 2:
p[0] = AST.Inits(p.lineno(1), p[1])
else:
p[0] = AST.Inits(p.lineno(1), p[1], p[3])
def p_init(self, p):
"""init : ID '=' expression """
p[0] = AST.Init(p.lineno(1), p[1], p[3])
def p_instructions(self, p):
"""instructions : instructions instruction
| instruction """
if len(p) == 3:
p[0] = AST.Instructions(p.lineno(1), p[1], p[2])
else:
p[0] = AST.Instructions(p.lineno(1), p[1])
def p_instruction(self, p):
"""instruction : print_instr
| labeled_instr
| assignment
| choice_instr
| while_instr
| repeat_instr
| return_instr
| break_instr
| continue_instr
| compound_instr"""
p[0] = p[1]
def p_print_instr(self, p):
"""print_instr : PRINT expression ';'
| PRINT error ';' """
if isinstance(p[2], AST.Expr):
p[0] = AST.PrintInstr(p.lineno(1), p[2])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
def p_labeled_instr(self, p):
"""labeled_instr : ID ':' instruction """
p[0] = AST.LabeledInstr(p.lineno(1), p[1], p[3])
def p_assignment(self, p):
"""assignment : ID '=' expression ';' """
p[0] = AST.Assignment(p.lineno(1), p[1], p[3])
def p_choice_instr(self, p):
"""choice_instr : IF '(' condition ')' instruction %prec IFX
| IF '(' condition ')' instruction ELSE instruction
| IF '(' error ')' instruction %prec IFX
| IF '(' error ')' instruction ELSE instruction """
if len(p) == 6:
if isinstance(p[3], AST.Condition):
p[0] = AST.IfInstr(p.lineno(1), p[3], p[5])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
else:
if isinstance(p[3], AST.Condition):
p[0] = AST.IfElseInstr(p.lineno(1), p[3], p[5], p[7])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
def p_while_instr(self, p):
"""while_instr : WHILE '(' condition ')' instruction
| WHILE '(' error ')' instruction """
if isinstance(p[3], AST.Condition):
p[0] = AST.WhileInstr(p.lineno(1), p[3], p[5])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
def p_repeat_instr(self, p):
"""repeat_instr : REPEAT instructions UNTIL condition ';' """
p[0] = AST.RepeatInstr(p.lineno(1), p[2], p[4])
def p_return_instr(self, p):
"""return_instr : RETURN expression ';' """
p[0] = AST.ReturnInstr(p.lineno(1), p[2])
def p_continue_instr(self, p):
"""continue_instr : CONTINUE ';' """
p[0] = AST.ContinueInstr(p.lineno(1))
def p_break_instr(self, p):
"""break_instr : BREAK ';' """
p[0] = AST.BreakInstr(p.lineno(1))
def p_compound_instr(self, p):
"""compound_instr : '{' declarations instructions '}' """
p[0] = AST.CompoundInstr(p.lineno(1), p[2], p[3])
def p_condition(self, p):
"""condition : expression"""
p[0] = AST.Condition(p.lineno(1), p[1])
def p_const(self, p):
"""const : INTEGER
| FLOAT
| STRING"""
if (p[1].isdigit()):
if "." in p[1]:
p[0] = AST.Float(p.lineno(1), p[1])
else:
p[0] = AST.Integer(p.lineno(1), p[1])
else:
p[0] = AST.String(p.lineno(1), p[1])
def p_expression(self, p):
"""expression : const
| ID
| expression '+' expression
| expression '-' expression
| expression '*' expression
| expression '/' expression
| expression '%' expression
| expression '|' expression
| expression '&' expression
| expression '^' expression
| expression AND expression
| expression OR expression
| expression SHL expression
| expression SHR expression
| expression EQ expression
| expression NEQ expression
| expression '>' expression
| expression '<' expression
| expression LE expression
| expression GE expression
| '(' expression ')'
| '(' error ')'
| ID '(' expr_list_or_empty ')'
| ID '(' error ')' """
if len(p) == 2:
p[0] = AST.UnExpr(p.lineno(1), p[1])
elif len(p) == 4:
if p[1] == '(':
if isinstance(p[2], AST.Expr):
p[0] = AST.BrackExpr(p.lineno(1), p[2])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
else:
p[0] = AST.BinExpr(p.lineno(1), p[2], p[1], p[3])
else:
if isinstance(p[3], AST.ExprList) or isinstance(p[3], AST.Epsilon):
p[0] = AST.FunCall(p.lineno(1), p[1], p[3])
else:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
sys.exit(-1)
def p_expr_list_or_empty(self, p):
"""expr_list_or_empty : expr_list
| """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = AST.Epsilon(p.lineno(0))
def p_expr_list(self, p):
"""expr_list : expr_list ',' expression
| expression """
if len(p) == 4:
p[0] = AST.ExprList(p.lineno(1), p[1], p[3])
else:
p[0] = AST.ExprList(p.lineno(1), p[1])
def p_fundefs(self, p):
"""fundefs : fundef fundefs
| """
if len(p) == 3:
p[0] = AST.FunDefs(p.lineno(1), p[1], p[2])
else:
p[0] = AST.Epsilon(p.lineno(0))
def p_fundef(self, p):
"""fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """
p[0] = AST.FunDef(p.lineno(1), p[1], p[2], p[4], p[6])
def p_args_list_or_empty(self, p):
"""args_list_or_empty : args_list
| """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = AST.Epsilon(p.lineno(0))
def p_args_list(self, p):
"""args_list : args_list ',' arg
| arg """
if len(p) == 4:
p[0] = AST.ArgsList(p.lineno(1), p[1], p[3])
else:
p[0] = AST.ArgsList(p.lineno(1), p[1])
def p_arg(self, p):
"""arg : TYPE ID """
p[0] = AST.Arg(p.lineno(1), p[2], p[1])
import json
from scanner import Scanner
scanner = Scanner()
i = 0
running = True
while running:
test = input("Start scan (y/n): ")
if test.lower() == 'y':
print("Scanning...")
json.dump(scanner.scan(), open('./output/{}.json'.format(i), 'w'))
print("Done: {}".format(i))
i += 1
elif test.lower() == 'n':
running = False
def new_scanner_numbers():
'''returns an instance of class Scanner'''
new_names = Names()
new_scanner = Scanner('test_files/test_numbers.txt', new_names)
return new_scanner_numbers
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
self.errorsOccured = False
def evaluate(self, sourceStr):
self._evaluator = PFEvaluator(Scanner(sourceStr))
value = self._evaluator.evaluate()
return value
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
class Cparser(object):
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
tokens = Scanner.tokens
precedence = (
("nonassoc", 'IFX'),
("nonassoc", 'ELSE'),
("right", '='),
("left", 'OR'),
("left", 'AND'),
("left", '|'),
("left", '^'),
("left", '&'),
("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'),
("left", 'SHL', 'SHR'),
("left", '+', '-'),
("left", '*', '/', '%'),
)
def p_error(self, p):
if p:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value))
else:
print("Unexpected end of input")
def p_program(self, p):
"""program : podprogramy"""
p[0] = AST.Program(p[1])
def p_podprogramy(self, p):
"""podprogramy : podprogramy podprogram
| """
if len(p) == 3:
podprogram = p[2]
if p[1] == None:
p[0] = AST.Podprogramy()
else:
p[0] = p[1]
p[0].addPodprogram(podprogram)
else:
p[0] = AST.Podprogramy()
def p_podprogram(self, p):
"""podprogram : declaration
| fundef
| instruction
| """
p[0] = AST.Podprogram(p[1])
def p_declaration(self, p):
"""declaration : TYPE inits ';'
| error ';' """
if len(p) == 4:
type = p[1]
inits = p[2]
p[0] = AST.Declaration(type, inits)
else:
p[0] = AST.Error(p[1])
def p_inits(self, p):
"""inits : inits ',' init
| init """
if len(p) == 4:
if p[1] == None:
p[0] = AST.InitList(p[3])
else:
p[0] = p[1]
p[0].addInit(p[3])
else:
p[0] = AST.InitList(p[1])
def p_init(self, p):
"""init : ID '=' expression """
id = p[1]
exp = p[3]
p[0] = AST.Init(id, exp, p.lineno(1))
def p_instructions(self, p):
"""instructions : instruction instructions
| instruction """
if len(p) == 3:
if p[2] == None:
p[0] = AST.InstList(p[1])
else:
p[0] = p[2]
p[0].addInst(p[1])
else:
p[0] = AST.InstList(p[1])
def p_instruction(self, p):
"""instruction : print_instr
| labeled_instr
| assignment
| choice_instr
| while_instr
| repeat_instr
| return_instr
| break_instr
| continue_instr
| compound_instr
| expression ';' """
p[0] = p[1]
def p_print_instr(self, p):
"""print_instr : PRINT expr_list ';'
| PRINT error ';' """
data = p[2]
p[0] = AST.PrtInst(data, p.lineno(1))
def p_labeled_instr(self, p):
"""labeled_instr : ID ':' instruction """
id = p[1]
inst = p[3]
p[0] = AST.LabInst(id, inst, p.lineno(1))
def p_assignment(self, p):
"""assignment : ID '=' expression ';' """
id = p[1]
exp = p[3]
p[0] = AST.AsgInst(id,exp, p.lineno(1))
def p_choice_instr(self, p):
"""choice_instr : IF '(' condition ')' instruction %prec IFX
| IF '(' condition ')' instruction ELSE instruction
| IF '(' error ')' instruction %prec IFX
| IF '(' error ')' instruction ELSE instruction """
cond = p[3]
inst = p[5]
if len(p) >= 8:
alt = p[7]
else:
alt = None
p[0] = AST.ChoiInst(cond,inst,alt)
def p_while_instr(self, p):
"""while_instr : WHILE '(' condition ')' instruction
| WHILE '(' error ')' instruction """
cond = p[3]
inst = p[5]
p[0] = AST.WhiInst(cond, inst)
def p_repeat_instr(self, p):
"""repeat_instr : REPEAT instructions UNTIL condition ';' """
inst = p[2]
cond = p[4]
p[0] = AST.RepInst(inst,cond)
def p_return_instr(self, p):
"""return_instr : RETURN expression ';' """
exp = p[2]
p[0] = AST.RetInst(exp, p.lineno(1))
def p_continue_instr(self, p):
"""continue_instr : CONTINUE ';' """
p[0] = AST.Continue(p.lineno(1))
def p_break_instr(self, p):
"""break_instr : BREAK ';' """
p[0] = AST.Break(p.lineno(1))
#Przypilnowac by instruction nie zawieral fundefu
#stworzyc unit -> declaration | instruction
#i jeszcze stworzyc kolejna co by miala liste unitow.
#poczatek zmian
def p_units(self, p):
"""units : units unit
| """
if len(p) == 3:
unit = p[2]
if p[1] == None:
p[0] = AST.Units()
else:
p[0] = p[1]
p[0].addUnit(unit)
else:
p[0] = AST.Units()
def p_unit(self, p):
"""unit : declaration
| instruction
| """
p[0] = AST.Unit(p[1]);
#koniec zmian
def p_compound_instr(self, p):
"""compound_instr : '{' units '}' """
if len(p[2].children) == 0:
p[0] = AST.CompInst(None, p.lineno(3))
else:
p[0] = AST.CompInst(p[2], p.lineno(3))
def p_condition(self, p):
"""condition : expression"""
p[0] = p[1]
def p_const(self, p):
"""const : integer
| float
| string"""
p[0] = p[1]
def p_integer(self, p):
"""integer : INTEGER"""
p[0] = AST.Integer(p[1], p.lineno(1))
def p_float(self, p):
"""float : FLOAT"""
p[0] = AST.Float(p[1], p.lineno(1))
def p_string(self, p):
"""string : STRING"""
text = filter(lambda c: c not in "\"", p[1])
p[0] = AST.String(text, p.lineno(1))
def p_expression_id(self, p):
"""expression : ID"""
p[0] = AST.Variable(p.lineno(1), p[1])
#H4X10R5K1 sposob na odronienie ID od const
def p_expression(self, p):
"""expression : const
| expression '+' expression
| expression '-' expression
| expression '*' expression
| expression '/' expression
| expression '%' expression
| expression '|' expression
| expression '&' expression
| expression '^' expression
| expression AND expression
| expression OR expression
| expression SHL expression
| expression SHR expression
| expression EQ expression
| expression NEQ expression
| expression '>' expression
| expression '<' expression
| expression LE expression
| expression GE expression
| '(' expression ')'
| '(' error ')'
| ID '(' expr_list_or_empty ')'
| ID '(' error ')' """
if len(p) == 2:
p[0] = p[1]
elif p[2] == '(' and p[1] != '(':
function_name = p[1]
parameters = p[3]
p[0] = AST.CallExp(function_name, parameters, p.lineno(1))
elif p[1] == '(':
p[0] = p[2]
else:
left = p[1]
operator = p[2]
right = p[3]
p[0] = AST.BinExpr(left, operator, right, p.lineno(2))
def p_expr_list_or_empty(self, p):
"""expr_list_or_empty : expr_list
| """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = None
def p_expr_list(self, p):
"""expr_list : expr_list ',' expression
| expression """
if len(p) == 4:
if p[1] == None:
p[0] = AST.ExpList(p[3])
else:
p[0] = p[1]
p[0].addExp(p[3])
else:
p[0] = AST.ExpList(p[1])
def p_fundef(self, p):
"""fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """
type = p[1]
name = p[2]
parameters = p[4]
body = p[6]
p[0] = AST.Function(type, name, parameters, body, p.lineno(1))
def p_args_list_or_empty(self, p):
"""args_list_or_empty : args_list
| """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = None
def p_args_list(self, p):
"""args_list : args_list ',' arg
| arg """
if len(p) == 4:
if p[3] == None:
p[0] = AST.ArgList(p[3])
else:
p[0] = p[1]
p[0].addArgToList(p[3])
else:
p[0] = AST.ArgList(p[1])
def p_arg(self, p):
"""arg : TYPE ID """
type = p[1]
name = p[2]
p[0] = AST.Parameter(type, name, p.lineno(1))
def new_scanner():
'''returns an instance of class Scanner'''
new_names = Names()
new_scanner = Scanner('test_files/ripple_counter.txt', new_names)
return new_scanner
def format(self, sourceStr):
normalizedStr = ""
scanner = Scanner(sourceStr);
while scanner.hasNext():
normalizedStr += str(scanner.next()) + " "
return normalizedStr;
class Parser(object):
def parse(self, sourceStr):
self._completionMessage = "No errors"
self._parseSuccessful = True
self._scanner = Scanner(sourceStr)
self._expression()
self._accept(self._scanner.get(), Token.EOE,
"symbol after end of expression")
def parseStatus(self):
return self._completionMessage
def _accept(self, token, expected, errorMessage):
if token.getType() != expected:
self._fatalError(token, errorMessage)
def _fatalError(self, token, errorMessage):
self._parseSuccessful = False
self._completionMessage = "Parsing error -- " + \
errorMessage + \
"\nExpression so far = " + \
self._scanner.stringUpToCurrentToken()
raise Exception, self._completionMessage
def _expression(self):
"""Syntax rule:
expression = term { addingOperator term } """
self._term()
token = self._scanner.get()
while token.getType() in (Token.PLUS, Token.MINUS):
self._scanner.next()
self._term()
token = self._scanner.get()
def _term(self):
"""Syntax rule:
term = factor { multiplyingOperator factor } """
self._factor()
token = self._scanner.get()
while token.getType() in (Token.MUL, Token.DIV):
self._scanner.next()
self._factor()
token = self._scanner.get()
def _factor(self):
"""Syntax rule:
factor = number | "(" expression ") """"
token = self._scanner.get()
if token.getType() == Token.INT:
self._scanner.next()
elif token.getType() == Token.L_PAR:
self._scanner.next()
self._expression()
self._accept(self._scanner.get(),
Token.R_PAR,
"')' expected")
self._scanner.next()
else:
self._fatalError(token, "bad factor")
print "buckets path:", buckets_dir
with open(index_path) as f:
reader = csv.reader(f)
items = list(reader)
if sum(map(bool, (args.scanner, args.max_order, args.word))) > 1:
parser.error(
'can only specify one of --word, --scanner and --max_order')
# Tokenize
if not SILENT:
print "will tokenize %d files" % len(items)
if args.scanner:
from scanner import Scanner
tokenizer = Scanner.from_file(args.scanner)
if not SILENT:
print "using provided scanner: ", args.scanner
elif args.word:
tokenizer = str.split
if not SILENT:
print "using str.split to tokenize"
else:
min_order = args.min_order if args.min_order else MIN_NGRAM_ORDER
max_order = args.max_order if args.max_order else MAX_NGRAM_ORDER
tokenizer = NGramTokenizer(min_order, max_order)
if not SILENT:
print "using n-gram tokenizer: min_order({0}) max_order({1})".format(
min_order, max_order)
if args.term_freq:
if not SILENT:
}
if char not in "".join(Token.get_operators()):
self.error("Unknown operator starting with '{}'".format(char))
next_char = self.scanner.get_next_char()
possible_op = char + next_char.char
# try to create a two-char operator; does it exist?
if (possible_op) in d.keys():
return Token(d[possible_op], possible_op)
# it is a one-char operator; store the 2nd char
else:
self.current_char = next_char
self.current_char_consumed = False
return Token(d[char], char)
def error(self, msg):
"""Raise an error with a custom message"""
raise Exception(msg)
if __name__ == "__main__":
while True:
text = input(">> ")
scan = Scanner(text)
Tok = Tokenizer(scan)
token = Tok.get_next_token()
print(token)
while token.get_type() is not Token.EOF:
token = Tok.get_next_token()
print(token)
def generate_xml(self, directory):
__scanner = Scanner(directory)
__scanner.generate_xml()
class Cparser(object):
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
self.errorsOccured = False
tokens = Scanner.tokens
precedence = (
("nonassoc", 'IFX'),
("nonassoc", 'ELSE'),
("right", '='),
("left", 'OR'),
("left", 'AND'),
("left", '|'),
("left", '^'),
("left", '&'),
("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'),
("left", 'SHL', 'SHR'),
("left", '+', '-'),
("left", '*', '/', '%'),
)
def p_error(self, p):
if p:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".
format(p.lineno, self.scanner.find_tok_column(p), p.type,
p.value))
else:
print('At end of input')
self.errorsOccured = True
def p_program(self, p):
"""program : classdefs declarations fundefs instructions"""
p[0] = AST.Program(p[1], p[2], p[3], p[4])
def p_declarations(self, p):
"""declarations : declarations declaration
| """
if len(p) > 1:
p[1].list.append(p[2])
p[0] = p[1]
else:
p[0] = AST.Declarations()
def p_declaration(self, p):
"""declaration : TYPE inits ';'
| id classinits ';'
| error ';' """
if len(p) > 2:
p[0] = AST.Declaration(p[1], p[2])
else:
p[0] = p[1]
def p_inits(self, p):
"""inits : inits ',' init
| init """
if len(p) > 2:
p[1].list.append(p[3])
p[0] = p[1]
else:
inits = AST.Inits()
inits.list.append(p[1])
p[0] = inits
def p_init(self, p):
"""init : id '=' expression """
p[0] = AST.Init(p[1], p[3])
def p_classinits(self, p):
"""classinits : classinits ',' classinit
| classinit """
if len(p) > 2:
p[1].list.append(p[3])
p[0] = p[1]
else:
classinits = AST.Classinits()
classinits.list.append(p[1])
p[0] = classinits
def p_classinit(self, p):
"""classinit : id """
p[0] = AST.Classinit(p[1])
def p_instructions(self, p):
"""instructions : instructions instruction
| instruction """
if len(p) == 2:
instructions = AST.Instructions()
instructions.list.append(p[1])
p[0] = instructions
else:
p[1].list.append(p[2])
p[0] = p[1]
def p_instruction(self, p):
"""instruction : expression ';'
| print_instr
| labeled_instr
| assignment
| choice_instr
| while_instr
| repeat_instr
| return_instr
| break_instr
| continue_instr
| compound_instr"""
p[0] = p[1]
def p_print_instr(self, p):
"""print_instr : PRINT expression ';'
| PRINT error ';' """
p[0] = AST.PrintInstr(p[2])
def p_labeled_instr(self, p):
"""labeled_instr : id ':' instruction """
p[0] = AST.LabeledInstr(p[1], p[3])
def p_assignment(self, p):
"""assignment : access '=' expression ';' """
p[0] = AST.Assignment(p[1], p[3])
def p_access(self, p):
"""access : id
| id '.' id """
if len(p) == 2:
access = AST.Access(p.lineno(1))
access.list.append(p[1])
p[0] = access
else:
access = AST.Access(p.lineno(1))
access.list.append(p[1])
access.list.append(p[3])
p[0] = access
def p_choice_instr(self, p):
"""choice_instr : IF '(' condition ')' instruction %prec IFX
| IF '(' condition ')' instruction ELSE instruction
| IF '(' error ')' instruction %prec IFX
| IF '(' error ')' instruction ELSE instruction """
if len(p) == 6:
p[0] = AST.ChoiceInstr(p[3], p[5], None)
else:
p[0] = AST.ChoiceInstr(p[3], p[5], p[7])
def p_while_instr(self, p):
"""while_instr : WHILE '(' condition ')' instruction
| WHILE '(' error ')' instruction """
p[0] = AST.WhileInstr(p[3], p[5])
def p_repeat_instr(self, p):
"""repeat_instr : REPEAT instructions UNTIL condition ';' """
p[0] = AST.RepeatInstr(p[2], p[4])
def p_return_instr(self, p):
"""return_instr : RETURN expression ';' """
p[0] = AST.ReturnInstr(p[2])
def p_continue_instr(self, p):
"""continue_instr : CONTINUE ';' """
p[0] = AST.Continue()
def p_break_instr(self, p):
"""break_instr : BREAK ';' """
p[0] = AST.Break()
def p_compound_instr(self, p):
"""compound_instr : '{' declarations instructions '}'"""
p[0] = AST.CompoundInstr(p[2], p[3])
def p_condition(self, p):
"""condition : expression"""
p[0] = AST.Condition(p[1])
def p_const(self, p):
"""const : integer
| float
| string"""
p[0] = p[1]
def p_integer(self, p):
"""integer : INTEGER"""
p[0] = AST.Integer(p[1], p.lineno(1))
def p_float(self, p):
"""float : FLOAT"""
p[0] = AST.Float(p[1], p.lineno(1))
def p_string(self, p):
"""string : STRING"""
p[0] = AST.String(p[1][1:-1], p.lineno(1))
def p_id(self, p):
"""id : ID"""
p[0] = AST.Id(p[1], p.lineno(1))
def p_expression(self, p):
"""expression : const
| access
| expression '+' expression
| expression '-' expression
| expression '*' expression
| expression '/' expression
| expression '%' expression
| expression '|' expression
| expression '&' expression
| expression '^' expression
| expression AND expression
| expression OR expression
| expression SHL expression
| expression SHR expression
| expression EQ expression
| expression NEQ expression
| expression '>' expression
| expression '<' expression
| expression LE expression
| expression GE expression
| '(' expression ')'
| '(' error ')'
| access '(' expr_list_or_empty ')'
| access '(' error ')' """
if len(p) == 2:
p[0] = p[1]
elif p[1] == '(':
p[0] = AST.ParExpr(p[2])
elif len(p) == 4:
p[0] = AST.BinExpr(p[1], p[2], p[3])
else:
p[0] = AST.FunExpr(p[1], p[3])
def p_expr_list_or_empty(self, p):
"""expr_list_or_empty : expr_list
| """
if len(p) == 1:
p[0] = AST.ExprList()
else:
p[0] = p[1]
def p_expr_list(self, p):
"""expr_list : expr_list ',' expression
| expression """
if len(p) == 4:
p[1].list.append(p[3])
p[0] = p[1]
else:
toReturn = AST.ExprList()
toReturn.list.append(p[1])
p[0] = toReturn
def p_fundefs(self, p):
"""fundefs : fundef fundefs
| """
if len(p) == 3:
p[2].list.reverse()
p[2].list.append(p[1])
p[2].list.reverse()
p[0] = p[2]
else:
p[0] = AST.FunDefs()
def p_fundef(self, p):
"""fundef : TYPE id '(' args_list_or_empty ')' compound_instr
| id id '(' args_list_or_empty ')' compound_instr"""
p[0] = AST.FunDef(p[1], p[2], p[4], p[6])
def p_args_list_or_empty(self, p):
"""args_list_or_empty : args_list
| """
if len(p) == 1:
p[0] = AST.ArgList()
else:
p[0] = p[1]
def p_args_list(self, p):
"""args_list : args_list ',' arg
| arg """
if len(p) == 2:
argList = AST.ArgList()
argList.list.append(p[1])
p[0] = argList
else:
p[1].list.append(p[3])
p[0] = p[1]
def p_arg(self, p):
"""arg : TYPE id
| id id"""
p[0] = AST.Arg(p[1], p[2])
def p_classdefs(self, p):
"""classdefs : classdef classdefs
| """
if len(p) == 3:
p[2].list.reverse()
p[2].list.append(p[1])
p[2].list.reverse()
p[0] = p[2]
else:
p[0] = AST.ClassDefs()
def p_classdef(self, p):
"""classdef : accessmodificator CLASS id classcontent
| accessmodificator CLASS id EXTENDS id classcontent"""
if len(p) < 7:
p[0] = AST.ClassDef(p[1], p[3], None, p[4])
else:
p[0] = AST.ClassDef(p[1], p[3], p[5], p[6])
def p_classcontent(self, p):
"""classcontent : '{' fielddefs ';' methoddefs '}' """
p[0] = AST.Classcontent(p[2], p[4])
def p_fielddefs(self, p):
"""fielddefs : fielddef fielddefs
| """
if len(p) == 3:
p[2].list.reverse()
p[2].list.append(p[1])
p[2].list.reverse()
p[0] = p[2]
else:
p[0] = AST.Fielddefs()
def p_fielddef(self, p):
"""fielddef : accessmodificator declaration """
p[0] = AST.Fielddef(p[1], p[2])
def p_methoddefs(self, p):
"""methoddefs : methoddef methoddefs
| """
if len(p) == 3:
p[2].list.reverse()
p[2].list.append(p[1])
p[2].list.reverse()
p[0] = p[2]
else:
p[0] = AST.Methoddefs()
def p_methoddef(self, p):
"""methoddef : accessmodificator fundef"""
p[0] = AST.Methoddef(p[1], p[2])
def p_accessmodificator(self, p):
"""accessmodificator : PRIVATE
| PROTECTED
| PUBLIC"""
p[0] = p[1]
def test_open_file_(names):
"""Test file open within scanner.py"""
f = open('test_specfiles/test_scanner/test_open_file.txt')
scanner = Scanner('test_specfiles/test_scanner/test_open_file.txt', names)
f1 = scanner._open_file(scanner.path)
assert f.read() == f1.read()
class Cparser(object):
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
tokens = Scanner.tokens
precedence = (
("nonassoc", 'IFX'),
("nonassoc", 'ELSE'),
("right", '='),
("left", 'OR'),
("left", 'AND'),
("left", '|'),
("left", '^'),
("left", '&'),
("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'),
("left", 'SHL', 'SHR'),
("left", '+', '-'),
("left", '*', '/', '%'),
)
def p_error(self, p):
if p:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".
format(p.lineno, self.scanner.find_tok_column(p), p.type,
p.value))
else:
print("Unexpected end of input")
def p_program(self, p):
"""program : blocks"""
p[0] = AST.Program(p[1])
TreePrinter()
print p[0]
def p_blocks(self, p):
"""blocks : blocks block
| """
if len(p) == 3:
p[0] = p[1] + [p[2]]
else:
p[0] = []
def p_block(self, p):
"""block : declaration
| instruction
| fundef"""
p[0] = AST.Block(p[1])
def p_declaration(self, p):
"""declaration : TYPE inits ';'
| error ';' """
if len(p) > 3:
p[0] = AST.Declaration(p[2])
def p_inits(self, p):
"""inits : inits ',' init
| init """
if len(p) == 4:
p[0] = p[1] + [p[3]]
else:
p[0] = [p[1]]
def p_init(self, p):
"""init : ID '=' expression """
p[0] = AST.Variable(p[1], p[3])
def p_instruction(self, p):
"""instruction : print_instr
| labeled_instr
| assignment
| choice_instr
| while_instr
| repeat_instr
| return_instr
| break_instr
| continue_instr
| compound_instr
| expression ';' """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = [p[1]]
def p_print_instr(self, p):
"""print_instr : PRINT expr_list ';'
| PRINT error ';' """
p[0] = [AST.Instruction(p[1], p[2])]
def p_labeled_instr(self, p):
"""labeled_instr : ID ':' instruction """
p[0] = [AST.Instruction(p[2], [p[1], p[3]])]
def p_assignment(self, p):
"""assignment : ID '=' expression ';' """
p[0] = [AST.Instruction(p[2], [p[1], p[3]])]
def p_choice_instr(self, p):
"""choice_instr : IF '(' condition ')' instruction %prec IFX
| IF '(' condition ')' instruction ELSE instruction
| IF '(' error ')' instruction %prec IFX
| IF '(' error ')' instruction ELSE instruction """
if len(p) == 6:
p[0] = [AST.Instruction(p[1], [p[3], p[5]])
] #Instruction(IF, [condition, instruction])
elif len(p) == 8:
p[0] = [
AST.Instruction(p[1], [p[3], p[5]]),
AST.Instruction(p[6], [p[7]])
] #Instruction(IF, [condition, instruction])
#Instruction(ELSE, [instruction])
def p_while_instr(self, p):
"""while_instr : WHILE '(' condition ')' instruction
| WHILE '(' error ')' instruction """
p[0] = [AST.Instruction(p[1], [p[3], p[5]])]
def p_repeat_instr(self, p):
"""repeat_instr : REPEAT instruction UNTIL condition ';' """
p[0] = [AST.Instruction(p[1], [p[2]]), AST.Instruction(p[3], [p[4]])]
def p_return_instr(self, p):
"""return_instr : RETURN expression ';' """
p[0] = [AST.Instruction(p[1], [p[2]])]
def p_continue_instr(self, p):
"""continue_instr : CONTINUE ';' """
p[0] = [AST.Instruction(p[1], [])]
def p_break_instr(self, p):
"""break_instr : BREAK ';' """
p[0] = [AST.Instruction(p[1], [])]
def p_compound_instr(self, p):
"""compound_instr : '{' comp_instr_blocks '}' """
p[0] = p[2]
def p_comp_instr_blocks(self, p):
"""comp_instr_blocks : comp_instr_blocks comp_instr_block
| """
if len(p) == 3:
p[0] = p[1] + [p[2]]
else:
p[0] = []
def p_comp_instr_block(self, p):
"""comp_instr_block : declaration
| instruction"""
p[0] = AST.Block(p[1])
def p_condition(self, p):
"""condition : expression"""
p[0] = p[1]
def p_const(self, p):
"""const : INTEGER
| FLOAT
| STRING"""
if type(p[1]) == int:
p[0] = AST.Const(AST.Integer(p[1]))
elif type(p[1]) == float:
p[0] = AST.Const(AST.Float(p[1]))
elif type(p[1]) == str:
p[0] = AST.Const(AST.String(p[1]))
def p_expression(self, p):
"""expression : const
| ID
| expression '+' expression
| expression '-' expression
| expression '*' expression
| expression '/' expression
| expression '%' expression
| expression '|' expression
| expression '&' expression
| expression '^' expression
| expression AND expression
| expression OR expression
| expression SHL expression
| expression SHR expression
| expression EQ expression
| expression NEQ expression
| expression '>' expression
| expression '<' expression
| expression LE expression
| expression GE expression
| '(' expression ')'
| '(' error ')'
| ID '(' expr_list_or_empty ')'
| ID '(' error ')' """
if len(p) == 2:
p[0] = p[1]
elif len(p) == 4:
if (p[1] == '('):
p[0] = p[2]
else:
p[0] = AST.BinExpr(p[2], p[1], p[3])
elif len(p) == 5:
p[0] = AST.Instruction('FUNCALL', [p[1], p[3]])
def p_expr_list_or_empty(self, p):
"""expr_list_or_empty : expr_list
| """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = []
def p_expr_list(self, p):
"""expr_list : expr_list ',' expression
| expression """
if len(p) == 4:
p[0] = p[1] + [p[3]]
else:
p[0] = [p[1]]
def p_fundef(self, p):
"""fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """
p[0] = AST.Fundef(p[1], p[2], p[4], p[6])
def p_args_list_or_empty(self, p):
"""args_list_or_empty : args_list
| """
if len(p) == 2:
p[0] = p[1]
else:
p[0] = []
def p_args_list(self, p):
"""args_list : args_list ',' arg
| arg """
if len(p) == 4:
p[0] = p[1] + [p[3]]
else:
p[0] = [p[1]]
def p_arg(self, p):
"""arg : TYPE ID """
p[0] = AST.Arg(p[1], p[2])
def scan(self, path):
with Scanner(path) as sc:
sc.go()
class Cparser(object):
def __init__(self):
self.error_occured = False
self.scanner = Scanner()
self.scanner.build()
tokens = Scanner.tokens
precedence = (
("nonassoc", 'IFX'),
("nonassoc", 'ELSE'),
("right", '='),
("left", 'OR'),
("left", 'AND'),
("left", '|'),
("left", '^'),
("left", '&'),
("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'),
("left", 'SHL', 'SHR'),
("left", '+', '-'),
("left", '*', '/', '%'),
)
def p_error(self, p):
if p:
print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".
format(p.lineno, self.scanner.find_tok_column(p), p.type,
p.value))
else:
print("Unexpected end of input")
def p_program(self, p):
"""program : main_parts_interlacing"""
#p[0] = ('program',p[1],p[2],p[3])
# print "{0}".format(p.lineno(1))
p[0] = Program(p[1])
def p_main_parts_interlacing(self, p):
"""main_parts_interlacing : main_parts_interlacing main_part
| main_part
"""
if p.__len__() == 3:
p[0] = MainPartsInerlacing(p[1], p[2])
else:
p[0] = MainPartsInerlacing(None, p[1])
def p_main_part(self, p):
"""
main_part : declaration
| fundef
| instruction
"""
p[0] = p[1]
def p_declarations(self, p):
"""declarations : declarations declaration
| """
#p[0] = ('') if p.__len__() == 1 else ('decl',p[1], p[2])
p[0] = None if p.__len__() == 1 else Declarations(p[1], p[2])
def p_declaration(self, p):
"""declaration : TYPE inits ';'
| error ';' """
#p[0] = ('declaration',p[1], p[2]) if p.__len__() == 4 else ('declaration-error',p[1])
#p[0] = Declaration(p[1], p[2])
if not type(p[1]) == LexToken: p[0] = Declaration(p[1], p[2])
else: self.error_occured = True
def p_inits(self, p):
"""inits : inits ',' init
| init """
#p[0] = ('inits',p[1],p[3]) if p.__len__() == 4 \
# else ('init',p[1])
p[0] = Inits(p[1],p[3]) if p.__len__() == 4 \
else p[1]
def p_init(self, p):
"""init : ID '=' expression """
#p[0] = ('init','=',p[1],p[3])
p[0] = Init(p[1], p[3])
def p_instructions_opt(self, p):
"""instructions_opt : instructions
| """
#p[0] = ('instructions_opt',p[1]) if p.__len__()==2 else('empty_instr')
p[0] = InstructionsOpt(p[1]) if p.__len__() == 2 else None
def p_instructions(self, p):
"""instructions : instructions instruction
| instruction """
#p[0] = ('instructions',p[1],p[2]) if p.__len__()==3 else ('instruction', p[1])
p[0] = Instructions(p[1], p[2]) if p.__len__() == 3 else p[1]
def p_instruction(self, p):
"""instruction : print_instr
| labeled_instr
| assignment
| choice_instr
| while_instr
| repeat_instr
| return_instr
| break_instr
| continue_instr
| compound_instr
| expression ';' """
#p[0] = ('instruction-node',p[1])
p[0] = p[1]
def p_print_instr(self, p):
"""print_instr : PRINT expr_list ';'
| PRINT error ';' """
#p[0] = ('print_instr', p[1], p[2])
if not type(p[2]) == LexToken: p[0] = PrintInstr(p[2])
else: self.error_occured = True
def p_labeled_instr(self, p):
"""labeled_instr : ID ':' instruction """
#p[0] = ('labeled_instr', p[1], p[3])
p[0] = LabeledInstr(p[1], p[3])
def p_assignment(self, p):
"""assignment : ID '=' expression ';' """
#p[0] = ('assignment',p[1],p[3])
p[0] = Assignment(p[1], p[3])
def p_choice_instr(self, p):
"""choice_instr : IF '(' condition ')' instruction %prec IFX
| IF '(' condition ')' instruction ELSE instruction
| IF '(' error ')' instruction %prec IFX
| IF '(' error ')' instruction ELSE instruction """
#p[0] = ('if-cond-instr',p[3],p[5])if p.__len__() == 6 else ('if-cond-instr-else-instr',p[3],p[5],p[7])
if not type(p[3]) == LexToken:
p[0] = ChoiceInstr(p[3],
p[5]) if p.__len__() == 6 else ChoiceInstr(
p[3], p[5], p[7])
else:
self.error_occured = True
def p_while_instr(self, p):
"""while_instr : WHILE '(' condition ')' instruction
| WHILE '(' error ')' instruction """
#p[0] = ('while-instr',p[3],p[5])
if not type(p[3]) == LexToken: p[0] = WhileInstr(p[3], p[5])
else: self.error_occured = True
def p_repeat_instr(self, p):
"""repeat_instr : REPEAT instructions UNTIL condition ';' """
#p[0] = ('repeat',p[2],p[4])
p[0] = RepeatInstr(p[2], p[4])
def p_return_instr(self, p):
"""return_instr : RETURN expression ';' """
#p[0] = ('return',p[2])
p[0] = ReturnInstr(p[2], p[1].lineno, p[1].column)
def p_continue_instr(self, p):
"""continue_instr : CONTINUE ';' """
#p[0] = ('continue')
p[0] = ContinueInstr(p[1].lineno, p[1].column)
def p_break_instr(self, p):
"""break_instr : BREAK ';' """
#p[0] = ('break')
p[0] = BreakInstr(p[1].lineno, p[1].column)
def p_compound_instr(self, p):
"""compound_instr : '{' declarations instructions_opt '}' """
#p[0] = ('compound_instr',p[2],p[3])
p[0] = CompoundInstr(p[2], p[3])
def p_condition(self, p):
"""condition : expression"""
#p[0] = ('condition',p[1])
p[0] = Condition(p[1])
def p_const(self, p):
"""const : INTEGER
| FLOAT
| STRING"""
#p[0] = p[1]
#p[0] = Const(p[1]) #to-do!!
p[0] = p[1]
def p_expression(self, p):
"""expression : const
| ID
| expression '+' expression
| expression '-' expression
| expression '*' expression
| expression '/' expression
| expression '%' expression
| expression '|' expression
| expression '&' expression
| expression '^' expression
| expression AND expression
| expression OR expression
| expression SHL expression
| expression SHR expression
| expression EQ expression
| expression NEQ expression
| expression '>' expression
| expression '<' expression
| expression LE expression
| expression GE expression
| '(' expression ')'
| '(' error ')'
| ID '(' expr_list_or_empty ')'
| ID '(' error ')' """
#p[0] = (p[1]) if p.__len__() == 2 else \
# (p[2],p[1],p[3]) if p.__len__() == 4 and p[1]!='(' else \
# p[2] if p.__len__() == 4 and p[1]=='(' else (p[1],p[3])
"""
p[0] = p[1] if p.__len__() == 2 else \
BinExpr(p[2],p[1],p[3]) if p.__len__() == 4 and p[1]!='(' else \
p[2] if p.__len__() == 4 and p[1]=='(' else DeclaredFunc(p[1],p[3])
"""
if p.__len__() == 2:
p[0] = p[1]
elif p.__len__() == 4:
if p[1] != '(':
p[0] = BinExpr(p[2], p[1], p[3])
else:
if not type(p[2]) == LexToken: p[0] = p[2]
else: self.error_occured = True
else:
if not type(p[3]) == LexToken: p[0] = DeclaredFunc(p[1], p[3])
else: self.error_occured = True
def p_expr_list_or_empty(self, p):
"""expr_list_or_empty : expr_list
| """
#p[0] = ('expr_list',p[1]) if p.__len__()==2 else ('empty_exprlist')
p[0] = PExprListOrEmpty(p[1]) if p.__len__() == 2 else None
def p_expr_list(self, p):
"""expr_list : expr_list ',' expression
| expression """
#p[0] = ('expr_list',p[1],p[3]) if p.__len__() == 4 else ('expr',p[1])
p[0] = PExprList(p[1], p[3]) if p.__len__() == 4 else p[1]
def p_fundefs_opt(self, p):
"""fundefs_opt : fundefs
| """
#p[0] = ('fundefs_opt',p[1]) if p.__len__() == 2 else ('no fundefs')
p[0] = FuncDefsOpt(p[1]) if p.__len__() == 2 else None
def p_fundefs(self, p):
"""fundefs : fundefs fundef
| fundef """
#p[0] = ('fundefs',p[1],p[2]) if p.__len__() == 3 else ("fundef", p[1])
p[0] = FuncDefs(p[1], p[2]) if p.__len__() == 3 else p[1]
def p_fundef(self, p):
"""fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """
#p[0] = ('fundef',p[2],p[1],p[4],p[6])
p[0] = FuncDef(p[1], p[2], p[4], p[6])
def p_args_list_or_empty(self, p):
"""args_list_or_empty : args_list
| """
#p[0] = ('args_list_',p[1]) if p.__len__() == 2 else ("empty-args-list")
p[0] = p[1] if p.__len__() == 2 else None
def p_args_list(self, p):
"""args_list : args_list ',' arg
| arg """
#p[0] = ('args_list',p[1],p[3]) if p.__len__() == 4 else ('arg', p[1])
p[0] = ArgsList(p[1], p[3]) if p.__len__() == 4 else p[1]
def p_arg(self, p):
"""arg : TYPE ID """
#p[0] = ('arg',p[1],p[2])
p[0] = Arg(p[1], p[2])
# !/bin/python
import sys
import ply.lex as lex
from scanner import Scanner
if __name__ == '__main__':
try:
filename = sys.argv[1] if len(
sys.argv) > 1 else "examples/scannerexample"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
text = file.read()
scanner = Scanner()
lexer = scanner.lexer
lexer.input(text) # Give the lexer some input
# Tokenize
while True:
tok = lexer.token()
if not tok:
break # No more input
column = scanner.find_column(tok)
print("(%d,%d): %s(%s)" % (tok.lineno, column, tok.type, tok.value))
from scanner import Scanner
scanner = Scanner('input.in', 'parser.automata')
'''
stringurile merg pe mai multe linii doar daca are \
'''
while True:
token = scanner.get_next_token()
if token is None:
break
print(token)
def __init__(self):
self.scanner = Scanner()
self.scanner.build()
self.args = {}
self.const = {}
import re
import os
import json
from scanner import Scanner
revp_ptn = re.compile(r'(\d+)_(\d+)')
revp_list = {}
data = {}
ret_ex = []
ret_vac = []
path = 'D:\github\data\shop_review'
uid_ptn = re.compile(r'href="/member/(\d+)(?:\?[^"]+)?"')
filename = 'uid.json'
s = Scanner(filename,
uid_ptn) # 找到每个店铺的评价页面的用户id，输出在uid.json中，key为店铺评价页面，values为对应页面的
# 评价的用户id
s.scan(path, save_period=500)
count = 0
rid = []
for k, v in s.data.items(): # 店铺评价页面的首页没有评价的店铺。输出在 rid.json中。
if len(v) == 0 and k.endswith('_1.html'):
count += 1
rid.append(k)
print count # 输出符合条件的店铺总数
with open("rid.json", 'w ') as fp:
json.dump(rid, fp)
for f in os.listdir(path): # 搜索目标文件夹中的文件名（店铺及评价页面），输出在revp.json中
class Parser(object):
def parse(self, sourceStr):
self._completionMessage = "No errors"
self._parseSuccessful = True
self._scanner = Scanner(sourceStr)
tree = self._expression()
self._accept(self._scanner.get(), Token.EOE,
"symbol after end of expression")
return tree
def parseStatus(self):
return self._completionMessage
def _accept(self, token, expected, errorMessage):
if token.getType() != expected:
self._fatalError(token, errorMessage)
def _fatalError(self, token, errorMessage):
self._parseSuccessful = False
self._completionMessage = "Parsing error -- " + \
errorMessage + \
"\nExpression so far = " + \
self._scanner.stringUpToCurrentToken()
raise Exception(self._completionMessage)
def _expression(self):
tree = self._term()
token = self._scanner.get()
while token.getType() in (Token.PLUS, Token.MINUS):
op = str(token)
self._scanner.next()
tree = InteriorNode(op, tree, self._term())
token = self._scanner.get()
return tree
def _term(self):
tree = self._factor()
token = self._scanner.get()
while token.getType() in (Token.MUL, Token.DIV):
op = str(token)
self._scanner.next()
tree = InteriorNode(op, tree, self._factor())
token = self._scanner.get()
return tree
def _factor(self):
tree = self._primary()
token = self._scanner.get()
if token.getType() == Token.EXPO:
op = str(token)
self._scanner.next()
tree = InteriorNode(op, tree, self._factor())
token = self._scanner.get()
return tree
def _primary(self):
token = self._scanner.get()
if token.getType() == Token.INT:
tree = LeafNode(token.getValue())
self._scanner.next()
elif token.getType() == Token.L_PAR:
self._scanner.next()
tree = self._expression()
self._accept(self._scanner.get(),
Token.R_PAR,
"')' expected")
self._scanner.next()
else:
tree = None
self._fatalError(token, "bad primary")
return tree
import click
from scanner import Scanner
from table import Table
import os
import time
from progress.bar import ChargingBar
table = Table()
scan = Scanner()
class Functions():
def __init__(self):
pass
@click.command()
@click.option('--nodes')
def view_status(nodes):
while True:
data = scan.run_scan(nodes)
os.system('cls')
tbl = table.render_table(
data, headers=["Address", "Status", "Time (ms)"])
print(tbl)
@click.command()
@click.option('--nodes')
def ip_scan(nodes):
data = scan.get_range(nodes)
bar = ChargingBar('Scanning', max=len(data))
discovered = []
for ip in data:
from scanner import Scanner
scanner = Scanner()
data = {}
import csv
verses = {}
multiple_matches = []
total_verses = 0
total_matches = 0
with open('data/verses.csv', 'rb') as csvfile:
versereader = csv.reader(csvfile, delimiter=',', quotechar='|')
count = 0
multiple_match_count = 0
for row in versereader:
(verse_id, input_string, real_scan) = row
scan = scanner.scan(input_string, known_only = True)
# if not(len(scan['results'])>0):
#
# print 'Error '+verse_id+":"+input_string
# scan = scanner.scan(input_string, known_only = False)
# print scanner.print_scan(scan)
# scan = scanner.scan(input_string, known_only = True)
total_matches+=len(scan['results'])
total_verses +=1
assert len(scan['results'])>0
print 'total verses = '+str(total_verses)
print 'total matches = '+str(total_matches)
assert total_verses==3314
