def buildMatrix(fname, rows, cols): # Construct an empty matrix with specified rows and cols matrix = [] for i in range(0, rows): matrixCols = [] for i in range(0, cols): matrixCols.append(0) matrix.append(matrixCols) # Open the file using Scanner or File Pointer # scanner example from http://troll.cs.ua.edu/cs150/projects/practice1.html if (fname): s = Scanner(fname) # Read the data from each row into an array scannedArray = [] token = s.readtoken() while token != '': scannedArray.append(token) token = s.readtoken() # Append the array onto the matrix for rowNumber in range(0, len(matrix)): for columnNumber in range(0, len(matrix[rowNumber])): if (len(scannedArray) > 0): matrix[rowNumber][columnNumber] = scannedArray.pop(0) # Close the file s.close() return matrix
def folder_scan(self, folders): class TimedProgressDisplay: def __init__(self, name, interval = 5): self.__name = name self.__interval = interval self.__last_display = 0 def __call__(self, scanned, total): if time.time() - self.__last_display > self.__interval or scanned == total: print "Scanning '{0}': {1}% ({2}/{3})".format(self.__name, (scanned * 100) / total, scanned, total) self.__last_display = time.time() s = Scanner(db.session) if folders: folders = map(lambda n: db.Folder.query.filter(db.Folder.name == n and db.Folder.root == True).first() or n, folders) if any(map(lambda f: isinstance(f, basestring), folders)): print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring)) for folder in filter(lambda f: isinstance(f, db.Folder), folders): FolderManager.scan(folder.id, s, TimedProgressDisplay(folder.name)) else: for folder in db.Folder.query.filter(db.Folder.root == True): FolderManager.scan(folder.id, s, TimedProgressDisplay(folder.name)) added, deleted = s.stats() db.session.commit() print "Scanning done" print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]) print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2])
def main(): args_handler = Handler() ip = args_handler.parsing_ip() tgt.send('1.1.1.1') scan = Scanner(tgt) scan.scan_ports()
class Parser: def __init__(self, text=''): self.lex = Scanner(text) # make a scanner self.vars = {'pi':3.14159} # add constants self.traceme = TraceDefault def parse(self, *text): # external interface if text: self.lex.newtext(text[0]) # reuse with new text tree = self.analyse() # parse string if tree: if self.traceme: # dump parse-tree? print; tree.trace(0) if self.errorCheck(tree): # check names self.interpret(tree) # evaluate tree def analyse(self): try: self.lex.scan() # get first token return self.Goal() # build a parse-tree except SyntaxError: print 'Syntax Error at column:', self.lex.start self.lex.showerror() except LexicalError: print 'Lexical Error at column:', self.lex.start self.lex.showerror() def errorCheck(self, tree): try: tree.validate(self.vars) # error checker return 'ok' except UndefinedError, varinfo: print "'%s' is undefined at column: %d" % varinfo self.lex.start = varinfo[1] self.lex.showerror() # returns None
def test_can_scan(self): self.skipTest('Don\'t know how to test with SSH :/') scanner = Scanner(network_tools=NetworkToolsStub(), hostname='10.0.1.231') scan = scanner.scan(port=9100) self.assertDictEqual(scan, {})
def run(sniffer_instance=None, wait_time=0.5, clear=True, args=(), debug=False): """ Runs the auto tester loop. Internally, the runner instanciates the sniffer_cls and scanner class. ``sniffer_instance`` The class to run. Usually this is set to but a subclass of scanner. Defaults to Sniffer. Sniffer class documentation for more information. ``wait_time`` The time, in seconds, to wait between polls. This is dependent on the underlying scanner implementation. OS-specific libraries may choose to ignore this parameter. Defaults to 0.5 seconds. ``clear`` Boolean. Set to True to clear the terminal before running the sniffer, (alias, the unit tests). Defaults to True. ``args`` The arguments to pass to the sniffer/test runner. Defaults to (). ``debug`` Boolean. Sets the scanner and sniffer in debug mode, printing more internal information. Defaults to False (and should usually be False). """ if sniffer_instance is None: sniffer_instance = ScentSniffer() if debug: scanner = Scanner(('.',), logger=sys.stdout) else: scanner = Scanner(('.',)) #sniffer = sniffer_cls(tuple(args), clear, debug) sniffer_instance.set_up(tuple(args), clear, debug) sniffer_instance.observe_scanner(scanner) scanner.loop(wait_time)
def compile_pascal(source, dest, is_debug = False, is_interpret = False, out_stream = sys.stdout, output_tokens = False, output_bytecodes = False, lib = ['.'], in_stream = sys.stdin): ''' DID YOU KNOW that compile() is a built in function? ''' set_debug(is_debug) debug("Compiling %s into %s" % (source, dest)) scanner = Scanner(source) tokens = scanner.scan() if output_tokens: write(tokens, source + "_tokenized") debug('scanning complete') parser = Parser(tokens, source, lib = lib) bytecodes, success = parser.parse() if output_bytecodes: if is_debug: write(prettify(bytecodes), source + "_unassembled") else: write(bytecodes, source + "_unassembled") if not success: print 'Parsing error' return debug('parsing complete') assembler = Assembler(bytecodes) assembled = assembler.assemble() if is_debug: write(prettify(assembled), dest + '_debug') write(assembled, dest) debug('assembly complete.' ) if is_interpret: interp = Interpreter(out_stream, in_stream, code = assembled) interp.interpret() else: debug('run program now with `python interpreter.py %s`' % dest)
def main(): ''' sets the structure for going through the dream process step by step ''' fullname = get_name(kind='in') # read in filename outname = get_name(kind='out') # read in output name opts = get_options() # read in options # start dream computations s = Scanner() # new scanner s.read_alpha_png(fullname) # read and store location of transparent pixels my_image = Image.open(fullname, 'r') # open image d = Dreamer(my_image) # create new dreamer with reference to image object img = iterate(d, my_image, outname, opts, save=False) # this function is for iterating over the same image multiple times out = s.cut_jpg(img) # send dream buffer object to be cut into transparent PNG new_image = Image.new('RGBA', my_image.size) # new blank image of same dimensions new_image.putdata(out) # make new image from the information from s.cut_jpg new_image.save(outname) # export image print("\nsaved image as: %s" % outname) open_image(outname) # give user option to automatically open image
def test_multipleinstances(self): report = Report.from_string(self.MULTIPLEINSTANCES) scanner = Scanner(report, [MultipleInstances()]) for ev in scanner.parse(): report.add_event(ev) self.logger.log(logging.INFO, "event: %s", str(ev)) self.assertEqual(report.count(), 1) report.report()
def test_build_fail_two(self): report = Report.from_string(self.BUILD_FAIL_2) scanner = Scanner(report, [BuildFail()]) for ev in scanner.parse(): report.add_event(ev) self.logger.log(logging.DEBUG, "event: %s", str(ev)) self.assertEqual(report.count(), 2) report.report()
def test_skipped_conflict(self): report = Report.from_string(self.SKIPPED_CONFLICT) scanner = Scanner(report, [SkippedConflict()]) for ev in scanner.parse(): report.add_event(ev) self.logger.log(logging.DEBUG, "event: %s", str(ev)) report.report() self.assertEqual(report.count(), 1)
class Tray(QtWidgets.QSystemTrayIcon): def __init__(self, client, icon, parent=None): QtWidgets.QSystemTrayIcon.__init__(self, icon, parent) self.client = client self.icon = icon self.user = self.client.get_account('me').url self.scanner = Scanner(self.client, self) self.options = OptionsWindow(self.client, self.scanner, self) self.options.init_options() self.stop_event = threading.Event() self.scan_thread = threading.Thread(target=self.scanner.scan, args=(self.stop_event,)) self.scan_thread.start() menu = QtWidgets.QMenu(parent) exitAction = QtWidgets.QAction("&Quit ", self) exitAction.setShortcut("Ctrl+Q") exitAction.setStatusTip('Good bye') exitAction.triggered.connect(self.appExit) optAction = QtWidgets.QAction("&Options... ", self) optAction.setShortcut("Ctrl+O") optAction.setStatusTip("Customize") optAction.triggered.connect(self.show_options) sendAction = QtWidgets.QAction("Copy Link of Last Uploaded Image", self) sendAction.setShortcut("Ctrl+S") sendAction.setStatusTip("...") sendAction.triggered.connect(self.copy_last) menu.addAction(sendAction) menu.addAction(optAction) menu.addSeparator() menu.addAction(exitAction) self.setContextMenu(menu) def appExit(self): # Reset OSX Screenshot storage with open(os.devnull, 'w') as nul: path = os.path.expanduser('~') + '/Desktop/' subprocess.call(["defaults", "write", "com.apple.screencapture", "location", path]) subprocess.call(["killAll", "SystemUIServer"]) kill_proc_tree(me) self.stop_event.set() self.scanner.scan(self.stop_event) sys.exit() def show_options(self): self.options.show_opts() def copy_last(self): if self.scanner.loader.link == '': self.scanner.loader.to_clipboard() else: self.scanner.loader.to_clipboard() self.scanner.loader.copy_notification()
def readTable(filename): s = Scanner(filename) table = [] record = readRecord(s) while record != "": table.append(record) record = readRecord(s) s.close() return table
def mainScanner(): file = open(filer) scanner = Scanner(file.read() + '\0') lexeme = scanner.scanNext() while lexeme.type != 200: lexeme.pprint() lexeme = scanner.scanNext()
def run(self): parsedArgs = self.options.parse() scanner = Scanner(parsedArgs.path) duplicateCollector = DuplicateCollector() scanner.scan(duplicateCollector) if parsedArgs.verbose: duplicateCollector.write(True) else: duplicateCollector.write()
def test_3(): s = Scanner() input_string = 'naqsh faryaadii hai kis kii sho;xii-e ta;hriir kaa' scan_results = s.scan(input_string, known_only=True) assert len(scan_results['results']) == 1 result = scan_results['results'][0] assert len(scan_results['results']) == 1 assert result['scan'] == '=-===-===-===-=', result['scan']
class MainController: def __init__(self, app): self.deviceData = DeviceData() self.view = MainView(None) self.view.scanForDevices.Bind(wx.EVT_BUTTON, self.ScanForDevices) self.view.syncActivities.Bind(wx.EVT_BUTTON, self.SyncActivities) self.view.Bind(wx.EVT_MENU, self.OnAbout, self.view.aboutMenuItem) self.view.Bind(wx.EVT_MENU, self.OnExit, self.view.exitMenuItem) self.view.Show() self.scanner = Scanner() ## TODO Preferences for Selected Scanners self.scanner.addScanner(AntScanner()) self.connector = Connector() ## TODO Preferences for Selected Connectors self.connector.addConnector(GarminConnector()) pub.subscribe(self.ScanningStarted, "SCANNING STARTED") pub.subscribe(self.DeviceDetected, "DEVICE DETECTED") pub.subscribe(self.ActivityRetrieved, "ACTIVITY RETRIEVED") pub.subscribe(self.ScanningEnded, "SCANNING ENDED") pub.subscribe(self.SyncStarted, "SYNC STARTED") pub.subscribe(self.SyncEnded, "SYNC ENDED") pub.subscribe(self.LoginSuccesful, "LOGIN SUCCESFUL") pub.subscribe(self.LoginFailed, "LOGIN FAILED") pub.subscribe(self.ActivitiesUploaded, "ACTIVITIES UPLOADED") def ScanForDevices(self, evt): self.scanner.scan() def ScanningStarted(self, evt): self.view.setStatus("Scanning started") def ScanningEnded(self, evt): self.view.setStatus("Scanning ended") def DeviceDetected(self, evt): self.view.setStatus("Device detected") def ActivityRetrieved(self, evt): self.view.setStatus("Retrieved activity") def SyncActivities(self, evt): self.connector.sync() def SyncStarted(self, evt): self.view.setStatus("Sync started") def SyncEnded(self, evt): self.view.setStatus("Sync ended") def LoginSuccesful(self, evt): self.view.setStatus("Login Succesful") def LoginFailed(self, evt): self.view.setStatus("Login Failed") def ActivitiesUploaded(self, evt): self.view.setStatus("Activities Uploaded") def OnExit(self,e): self.Close(True) def OnAbout(self, event): dlg = wx.MessageDialog( self.view, "A community-developed Linux version of the ANT Agent. Supports Garmin-based fitness devices that communicate either over USB serial or via the ANT USB connector. Developed by Philip Whitehouse, based on work by Braiden Kindt, Gustav Tiger and Collin (cpfair). Copyright 2014", "About ANT Agent for Linux", wx.OK); dlg.ShowModal() dlg.Destroy()
def test_01(): text_XX = """ @@ aa """ def callback(mo): D("callback %r", mo) scanner = Scanner(( (r'^@@', callback), (r'aaa', callback), ), re.MULTILINE) scanner.scan(text_XX)
def run_parser(self, file_, lib_classes = None): """parses a jml file and returns a list of abstract syntax trees, or parses a string containing JaML code. """ scanner = Scanner() # Try and parse a file, if this fails, parse as a string try: # This is to see if it's a file or not open(file_) # Stores the directory of the main class dir_ = os.path.dirname(file_) # Add the name of the main class to begin with file_name = os.path.basename(file_) class_name = file_name.replace('.jml', '') # this stores names of all already seen class references seen = [class_name] # Stores classes to parse to_parse = [class_name] # Stores the ASTs of parsed classes parsed = nodes.ProgramASTs() while to_parse: # Append file information to the class names file_name = os.path.join(dir_, to_parse[0] + '.jml') # Get the raw input raw = open(file_name) input_ = raw.read() # Scan and parse the file tokens = scanner.tokenize(input_) ast = self.parse(tokens) # Check the class and file are named the same if to_parse[0] != ast.children[0].value: msg = 'Class name and file name do not match!' raise NameError(msg) to_parse.pop(0) parsed.append(ast) # Fined classes reference from the one just parsed if lib_classes == None: lib_classes = {} refed_classes = self._find_refed_classes refed_classes = refed_classes(ast, seen, [], lib_classes, dir_) seen += refed_classes to_parse += refed_classes return parsed except IOError: # Simply parse the program held in the string tokens = scanner.tokenize(file_) ast = self.parse(tokens) ast_wrapper = nodes.ProgramASTs() ast_wrapper.append(ast) return ast_wrapper
def main(argv): if len(argv) == 1 or len(argv) > 2: usage() sys.exit(2) try: path = argv[1] scanner = Scanner() scanner.scan(path) scanner.rebuild_tracks() fcpxml = FcpXML() fcpxml.create_xml() except getopt.GetoptError, err: print str(err) usage() sys.exit(2)
def main(): # Include current path in case we attempt to load modules. sys.path.append(os.getcwd()) parser = argparse.ArgumentParser("Runs a set of tests against the set of provided URLs") parser.add_argument("-u", "--url", action="append", dest="targets", help="Add a target to test") parser.add_argument("-f", "--target-file", action="append", dest="target_files", help="File with URLs to test") parser.add_argument("-m", "--module", action="append", default = [], dest="modules", help="Load an extension module") parser.add_argument("-D", "--disable-core", action="store_true", default = False, dest="disable_core", help="Disable corechecks") parser.add_argument("-p", "--force-passive", action="store_true", default=False, dest="force_passives", help ="Force passives to be run for each active test") parser.add_argument("-d", "--dns", action="store_false", default=True, dest="resolve_target", help ="Skip DNS resolution when registering a target") parser.add_argument("-r", "--report", action="store", default="xml", dest="report",help="Load a reporter e.g. -r reporter.AntXmlReporter") parser.add_argument("-o", "--output", action="store", default="garmr-results.xml", dest="output", help="Default output is garmr-results.xml") parser.add_argument("-c", "--check", action="append", dest="opts", help="Set a parameter for a check (check:opt=value)" ) parser.add_argument("-e", "--exclude", action="append", dest="exclusions", help="Prevent a check from being run/processed") parser.add_argument("--save", action="store", dest="dump_path", help="Write out a configuration file based on parameters (won't run scan)") args = parser.parse_args() scanner = Scanner() scanner.force_passives = args.force_passives scanner.resolve_target = args.resolve_target scanner.output = args.output # Start building target list. if args.targets != None: for target in args.targets: scanner.register_target(target) # Add targets from files to the list. if args.target_files != None: for targets in args.target_files: try: f = open(targets, "r") for target in f: t = target.strip() if len(t) > 0: scanner.register_target(t) except: Scanner.logger.error("Unable to process the target list in: %s", targets) # Load built-in modules if required. if args.disable_core == False: corechecks.configure(scanner) # Configure modules. # TODO: change the module loading to scan the list of classes in a module and automagically # detect any tests defined. if args.modules != None: for module in args.modules: try: __import__(module) m = sys.modules[module] m.configure(scanner) except Exception, e: Scanner.logger.fatal("Unable to load the requested module [%s]: %s", module, e) quit()
def parse(self, sourceStr): self._completionMessage = "No errors" self._parseSuccessful = True self._scanner = Scanner(sourceStr) self._expression() self._accept(self._scanner.get(), Token.EOE, "symbol after end of expression")
def folder_prune(): s = Scanner(session) folders = session.query(Folder).filter(Folder.root == True) if folders: for folder in folders: print "Pruning: " + folder.path FolderManager.prune(folder.id, s) added, deleted = s.stats() print "\a" print "Pruning done" print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]) print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2])
def __init__(self, client, icon, parent=None): QtWidgets.QSystemTrayIcon.__init__(self, icon, parent) self.client = client self.icon = icon self.user = self.client.get_account('me').url self.scanner = Scanner(self.client, self) self.options = OptionsWindow(self.client, self.scanner, self) self.options.init_options() self.stop_event = threading.Event() self.scan_thread = threading.Thread(target=self.scanner.scan, args=(self.stop_event,)) self.scan_thread.start() menu = QtWidgets.QMenu(parent) exitAction = QtWidgets.QAction("&Quit ", self) exitAction.setShortcut("Ctrl+Q") exitAction.setStatusTip('Good bye') exitAction.triggered.connect(self.appExit) optAction = QtWidgets.QAction("&Options... ", self) optAction.setShortcut("Ctrl+O") optAction.setStatusTip("Customize") optAction.triggered.connect(self.show_options) sendAction = QtWidgets.QAction("Copy Link of Last Uploaded Image", self) sendAction.setShortcut("Ctrl+S") sendAction.setStatusTip("...") sendAction.triggered.connect(self.copy_last) menu.addAction(sendAction) menu.addAction(optAction) menu.addSeparator() menu.addAction(exitAction) self.setContextMenu(menu)
def createSearchArray(size): command = str.format("python3 makeintegers.py {0} 0 1 {1} > data.out", size, swaps) # create a file called data.out with the randomized array subprocess.call(command, shell=True) numFile = Scanner("data.out") searchArray = DynamicArray(size) # create a DynamicArray with the randomized search data while True: token = numFile.readint() if token == "": break else: searchArray.addToBack(token) return searchArray
def run(cls): s3 = boto.connect_s3() prod_bucket = s3.get_bucket('dray-assets-prod') load_files = list(prod_bucket.list(prefix="media/loads")) document_files = [f for f in load_files if "load_action_approvals" not in f.name] for f in document_files: # Check for existing scanned version filename, file_extension = os.path.splitext(f.key) result_keyname = filename + "-scanned" + file_extension possible_key = prod_bucket.get_key(result_keyname) if possible_key is not None or "-scanned" in f.key: # Scanned file already exists, move on continue # Read image and scan image = prod_bucket.get_key(f.key) image.get_contents_to_filename('tmp/tmp-src.png') result_filename = Scanner.scan('tmp/tmp-src.png') # Create new key for result file k = Key(prod_bucket) k.key = result_keyname k.set_contents_from_filename(result_filename)
def scan_folder(id = None): s = Scanner(session) if id is None: for folder in Folder.query.filter(Folder.root == True): FolderManager.scan(folder.id, s) else: status = FolderManager.scan(id, s) if status != FolderManager.SUCCESS: flash(FolderManager.error_str(status)) return redirect(url_for('folder_index')) added, deleted = s.stats() session.commit() flash('Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2])) flash('Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2])) return redirect(url_for('folder_index'))
def __init__(self, settings): """TODO: to be defined1. """ self.cubo = Cubo() self.cuboescaneado = CuboEscaneado() self.settings = settings self.scanner = Scanner(self.cubo, self.settings) self.robot = Robot() self.solver = Solver(self.cubo)
def handle(self): from scanner import Scanner s = Scanner() s.check_directories() s.initialize_logger() s.check_yara_file() # Check in case client terminates connection mid xfer self.request.settimeout(10.0) try: raw_msg_len = self.request.recv(4) msg_len = struct.unpack('>I', raw_msg_len)[0] data = '' # Basic client request integrity check, not fullproof proto_check = self.request.recv(7) if proto_check != 'FSF_RPC': s.dbg_h.error('%s Client request integrity check failed. Invalid FSF protocol used.' % dt.now()) raise ValueError() while len(data) < msg_len: recv_buff = self.request.recv(msg_len - len(data)) data += recv_buff self.request.settimeout(None) self.process_data(data, s) except: e = sys.exc_info()[0] s.dbg_h.error('%s There was a problem processing the connection request from %s. Error: %s' % (dt.now(), self.request.getpeername()[0], e)) finally: self.request.close()
def handle(self): from scanner import Scanner s = Scanner() s.check_directories() s.initialize_logger() s.check_yara_file() try: raw_msg_len = self.request.recv(4) msg_len = struct.unpack('>I', raw_msg_len)[0] data = '' while len(data) < msg_len: recv_buff = self.request.recv(msg_len - len(data)) data += recv_buff self.process_data(data, s) except: e = sys.exc_info()[0] s.dbg_h.error('%s There was a problem processing the connection request from %s. Error: %s' % (dt.now(), self.request.getpeername()[0], e)) finally: self.request.close()
or stack.peek().getPrecedence() < token.getPrecedence()): break self._pfTokens.append(stack.pop()) stack.push(token) for i in range(len(stack)): self._pfTokens.append(stack.pop()) return self._pfTokens def __inter__(self): if not self._pfTokens: self.convert() return iter(self._pfTokens) def __str__(self): if not self._pfTokens: self.convert() return ' '.join(map(str, self._pfTokens)) if __name__ == '__main__': ife = '(4+5)*(6-3)' scanner = Scanner(ife) print('Infix expression:', scanner) converter = IFToPFConverter(scanner) tokenList = converter.convert() print(*tokenList) print(converter)
args = sys.argv if len(args) > 2: print("Usage: ./lox [script]") sys.exit(64) from scanner import Scanner from parser import Parser, ParseError from interpreter import Interpreter from resolver import Resolver if len(args) == 2: filename = args[1] with open(filename, "r") as f: data = f.read() tokens = Scanner(data).scan_tokens() statements = Parser(tokens).parse() if not Lox.had_error: interpreter = Interpreter() Resolver(interpreter).resolve(statements) if not Lox.had_error: interpreter.interpret(statements) else: print("Lox 0.1.0") interpreter = Interpreter() try: while True: line = input("> ") tokens = Scanner(line).scan_tokens() statements = Parser(tokens).parse() if not Lox.had_error:
class Mparser(object): def __init__(self): self.scanner = Scanner() self.scanner.build() self.args = {} self.const = {} tokens = Scanner.tokens precedence = ( ("nonassoc", 'IFX'), ("nonassoc", 'ELSE'), ("nonassoc", 'LT', 'GT', 'EQ', 'NE', 'LE', 'GE'), ("left", '+', '-'), ("left", '*', '/'), ("left", 'DOTADD', 'DOTSUB'), ("left", 'DOTMUL', 'DOTDIV'), ) def p_error(self, p): if p: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')". format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) else: print('At end of input') def p_program(self, p): """program : instructions""" p[0] = AST.Program(p[1]) print(p[0]) def p_instructions(self, p): """instructions : instructions instruction | instruction """ if len(p) == 3: p[1].instrs.append(p[2]) p[0] = p[1] else: p[0] = AST.Instructions(p[1]) def p_instruction(self, p): """instruction : print_instr | assignment | choice_instr | while_instr | for_instr | return_instr | break_instr | continue_instr | compound_instr""" p[0] = p[1] def p_print_instr(self, p): """print_instr : PRINT expr_list ';' | PRINT error ';' """ p[0] = AST.Print(p[2]) def p_assignment(self, p): """assignment : id assign_operator expression ';' | ref assign_operator expression ';' """ p[0] = AST.Assignment(p[1], p[2], p[3]) def p_ref(self, p): """ref : id '[' expr_list ']' """ p[0] = AST.Ref(p[1], p[3]) def p_assign_operator(self, p): """assign_operator : '=' | ADDASSIGN | SUBASSIGN | MULASSIGN | DIVASSIGN""" p[0] = AST.Assign_operator(p[1]) def p_vector(self, p): """vector : '[' expressions ']' """ p[0] = AST.Vector(p[2]) def p_expressions_or_vectors(self, p): """expressions : expression | expressions ',' expression """ if len(p) == 4: p[1].exprs.append(p[3]) p[0] = p[1] else: p[0] = AST.Expressions(p[1]) def p_choice_instr(self, p): """choice_instr : IF '(' condition ')' instruction %prec IFX | IF '(' condition ')' instruction ELSE instruction | IF '(' error ')' instruction %prec IFX | IF '(' error ')' instruction ELSE instruction """ if len(p) == 6: p[0] = AST.Choice(p[3], p[5]) else: p[0] = AST.Choice(p[3], p[5], p[7]) def p_while_instr(self, p): """while_instr : WHILE '(' condition ')' instruction | WHILE '(' error ')' instruction """ p[0] = AST.While(p[3], p[5]) def p_for_instr(self, p): """for_instr : FOR id '=' range instruction""" p[0] = AST.For(p[2], p[4], p[5]) def p_range(self, p): """range : expression ':' expression""" p[0] = AST.Range(p[1], p[3]) def p_return_instr(self, p): """return_instr : RETURN expression ';' """ p[0] = AST.Return(p[2]) def p_continue_instr(self, p): """continue_instr : CONTINUE ';' """ p[0] = AST.Continue() def p_break_instr(self, p): """break_instr : BREAK ';' """ p[0] = AST.Break() def p_compound_instr(self, p): """compound_instr : '{' instructions '}' """ p[0] = AST.ComInstructions(p[2]) def p_condition(self, p): """condition : expression""" p[0] = p[1] def p_const(self, p): """const : INTNUM | FLOATNUM | STRING""" p[0] = AST.Const(p[1]) def p_expression(self, p): """expression : const | id | ref | vector | matrix_operation | matrix_function | minus_matrix | matrix_transposed | expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression EQ expression | expression NE expression | expression LT expression | expression GT expression | expression LE expression | expression GE expression | '(' expression ')' | '(' error ')'""" if len(p) == 2: p[0] = p[1] elif len(p) == 4: if p[1] == '(': p[0] = p[2] else: if p[2] in ['+', '-', '*', '/']: p[0] = AST.BinExpr(p[1], p[2], p[3]) else: p[0] = AST.Condition(p[1], p[2], p[3]) def p_id(self, p): """id : ID""" p[0] = AST.Variable(p[1], 0) def p_matrix_operation(self, p): """matrix_operation : matrix dot_operation matrix""" p[0] = AST.Matrix_operation(p[1], p[2], p[3]) def p_dot_operation(self, p): """dot_operation : DOTADD | DOTSUB | DOTMUL | DOTDIV""" p[0] = AST.Dot_operation(p[1]) def p_matrix(self, p): """matrix : id | minus_matrix | matrix_transposed""" p[0] = AST.Matrix(p[1]) def p_matrix_transposed(self, p): """matrix_transposed : matrix "'" """ p[0] = AST.Matrix_transposed(p[1]) def p_minus_matrix(self, p): """minus_matrix : "-" matrix """ p[0] = AST.Minus_matrix(p[2]) def p_matrix_function(self, p): """matrix_function : ZEROS '(' expressions ')' | ONES '(' expressions ')' | EYE '(' expressions ')' """ p[0] = AST.Matrix_function(p[1], p[3]) def p_expr_list(self, p): """expr_list : expr_list ',' expression | expression """ if len(p) == 4: p[1].exprs.append(p[3]) p[0] = p[1] else: p[0] = AST.Expressions(p[1])
self.look_ahead].startswith('boollit') or self.tokens[ self.look_ahead] == 'LP': term.add_child(self.factor()) if self.tokens[self.look_ahead].startswith('MULTIPLICATIVE'): term.add_child(self.match('MULTIPLICATIVE')) term.add_child(self.term()) return term def factor(self): factor = Node('<factor>') if self.tokens[self.look_ahead].startswith('ident'): factor.add_child(self.match('ident')) elif self.tokens[self.look_ahead].startswith('num'): factor.add_child(self.match('num')) elif self.tokens[self.look_ahead].startswith('boollit'): factor.add_child(self.match('boollit')) elif self.tokens[self.look_ahead] == 'LP': factor.add_child(self.match('LP')) factor.add_child(self.simple_expression()) factor.add_child(self.match('RP')) return factor if __name__ == '__main__': scanner = Scanner() parser = Parser(scanner) parser.parse()
from checker import Checker from scanner import Scanner from parser import Parser from encoder import Encoder fileDir = './example_files/prog1.txt' if __name__ == "__main__": scanner = Scanner(fileDir) parser = Parser(scanner) checker = Checker() encoder = Encoder() program = parser.parse_program() checker.check(program) encoder.encode(program) encoder.save_target_program('examples_files/prog1.tam') # print(json.dumps(program, default=lambda x: x.__dict__, indent=2))
class Cparser(object): def __init__(self): self.scanner = Scanner() self.scanner.build() tokens = Scanner.tokens precedence = ( ("nonassoc", 'IFX'), ("nonassoc", 'ELSE'), ("right", '='), ("left", 'OR'), ("left", 'AND'), ("left", '|'), ("left", '^'), ("left", '&'), ("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'), ("left", 'SHL', 'SHR'), ("left", '+', '-'), ("left", '*', '/', '%'), ) def p_error(self, p): if p: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) else: print('At end of input') def p_program(self, p): """program : declarations fundefs instructions""" p[0] = AST.Program(p.lineno(1), p[1], p[2], p[3]) print(p[0]) def p_declarations(self, p): """declarations : declarations declaration | """ if len(p) == 3: p[0] = AST.Declarations(p.lineno(1), p[1], p[2]) else: p[0] = AST.Epsilon(p.lineno(0)) def p_declaration(self, p): """declaration : TYPE inits ';' | error ';' """ if len(p) == 4: p[0] = AST.Declaration(p.lineno(1), p[2], p[1]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) def p_inits(self, p): """inits : inits ',' init | init """ if len(p) == 2: p[0] = AST.Inits(p.lineno(1), p[1]) else: p[0] = AST.Inits(p.lineno(1), p[1], p[3]) def p_init(self, p): """init : ID '=' expression """ p[0] = AST.Init(p.lineno(1), p[1], p[3]) def p_instructions(self, p): """instructions : instructions instruction | instruction """ if len(p) == 3: p[0] = AST.Instructions(p.lineno(1), p[1], p[2]) else: p[0] = AST.Instructions(p.lineno(1), p[1]) def p_instruction(self, p): """instruction : print_instr | labeled_instr | assignment | choice_instr | while_instr | repeat_instr | return_instr | break_instr | continue_instr | compound_instr""" p[0] = p[1] def p_print_instr(self, p): """print_instr : PRINT expression ';' | PRINT error ';' """ if isinstance(p[2], AST.Expr): p[0] = AST.PrintInstr(p.lineno(1), p[2]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) def p_labeled_instr(self, p): """labeled_instr : ID ':' instruction """ p[0] = AST.LabeledInstr(p.lineno(1), p[1], p[3]) def p_assignment(self, p): """assignment : ID '=' expression ';' """ p[0] = AST.Assignment(p.lineno(1), p[1], p[3]) def p_choice_instr(self, p): """choice_instr : IF '(' condition ')' instruction %prec IFX | IF '(' condition ')' instruction ELSE instruction | IF '(' error ')' instruction %prec IFX | IF '(' error ')' instruction ELSE instruction """ if len(p) == 6: if isinstance(p[3], AST.Condition): p[0] = AST.IfInstr(p.lineno(1), p[3], p[5]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) else: if isinstance(p[3], AST.Condition): p[0] = AST.IfElseInstr(p.lineno(1), p[3], p[5], p[7]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) def p_while_instr(self, p): """while_instr : WHILE '(' condition ')' instruction | WHILE '(' error ')' instruction """ if isinstance(p[3], AST.Condition): p[0] = AST.WhileInstr(p.lineno(1), p[3], p[5]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) def p_repeat_instr(self, p): """repeat_instr : REPEAT instructions UNTIL condition ';' """ p[0] = AST.RepeatInstr(p.lineno(1), p[2], p[4]) def p_return_instr(self, p): """return_instr : RETURN expression ';' """ p[0] = AST.ReturnInstr(p.lineno(1), p[2]) def p_continue_instr(self, p): """continue_instr : CONTINUE ';' """ p[0] = AST.ContinueInstr(p.lineno(1)) def p_break_instr(self, p): """break_instr : BREAK ';' """ p[0] = AST.BreakInstr(p.lineno(1)) def p_compound_instr(self, p): """compound_instr : '{' declarations instructions '}' """ p[0] = AST.CompoundInstr(p.lineno(1), p[2], p[3]) def p_condition(self, p): """condition : expression""" p[0] = AST.Condition(p.lineno(1), p[1]) def p_const(self, p): """const : INTEGER | FLOAT | STRING""" if (p[1].isdigit()): if "." in p[1]: p[0] = AST.Float(p.lineno(1), p[1]) else: p[0] = AST.Integer(p.lineno(1), p[1]) else: p[0] = AST.String(p.lineno(1), p[1]) def p_expression(self, p): """expression : const | ID | expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression '%' expression | expression '|' expression | expression '&' expression | expression '^' expression | expression AND expression | expression OR expression | expression SHL expression | expression SHR expression | expression EQ expression | expression NEQ expression | expression '>' expression | expression '<' expression | expression LE expression | expression GE expression | '(' expression ')' | '(' error ')' | ID '(' expr_list_or_empty ')' | ID '(' error ')' """ if len(p) == 2: p[0] = AST.UnExpr(p.lineno(1), p[1]) elif len(p) == 4: if p[1] == '(': if isinstance(p[2], AST.Expr): p[0] = AST.BrackExpr(p.lineno(1), p[2]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) else: p[0] = AST.BinExpr(p.lineno(1), p[2], p[1], p[3]) else: if isinstance(p[3], AST.ExprList) or isinstance(p[3], AST.Epsilon): p[0] = AST.FunCall(p.lineno(1), p[1], p[3]) else: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) sys.exit(-1) def p_expr_list_or_empty(self, p): """expr_list_or_empty : expr_list | """ if len(p) == 2: p[0] = p[1] else: p[0] = AST.Epsilon(p.lineno(0)) def p_expr_list(self, p): """expr_list : expr_list ',' expression | expression """ if len(p) == 4: p[0] = AST.ExprList(p.lineno(1), p[1], p[3]) else: p[0] = AST.ExprList(p.lineno(1), p[1]) def p_fundefs(self, p): """fundefs : fundef fundefs | """ if len(p) == 3: p[0] = AST.FunDefs(p.lineno(1), p[1], p[2]) else: p[0] = AST.Epsilon(p.lineno(0)) def p_fundef(self, p): """fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """ p[0] = AST.FunDef(p.lineno(1), p[1], p[2], p[4], p[6]) def p_args_list_or_empty(self, p): """args_list_or_empty : args_list | """ if len(p) == 2: p[0] = p[1] else: p[0] = AST.Epsilon(p.lineno(0)) def p_args_list(self, p): """args_list : args_list ',' arg | arg """ if len(p) == 4: p[0] = AST.ArgsList(p.lineno(1), p[1], p[3]) else: p[0] = AST.ArgsList(p.lineno(1), p[1]) def p_arg(self, p): """arg : TYPE ID """ p[0] = AST.Arg(p.lineno(1), p[2], p[1])
import json from scanner import Scanner scanner = Scanner() i = 0 running = True while running: test = input("Start scan (y/n): ") if test.lower() == 'y': print("Scanning...") json.dump(scanner.scan(), open('./output/{}.json'.format(i), 'w')) print("Done: {}".format(i)) i += 1 elif test.lower() == 'n': running = False
def new_scanner_numbers(): '''returns an instance of class Scanner''' new_names = Names() new_scanner = Scanner('test_files/test_numbers.txt', new_names) return new_scanner_numbers
def __init__(self): self.scanner = Scanner() self.scanner.build() self.errorsOccured = False
def evaluate(self, sourceStr): self._evaluator = PFEvaluator(Scanner(sourceStr)) value = self._evaluator.evaluate() return value
def __init__(self): self.scanner = Scanner() self.scanner.build()
class Cparser(object): def __init__(self): self.scanner = Scanner() self.scanner.build() tokens = Scanner.tokens precedence = ( ("nonassoc", 'IFX'), ("nonassoc", 'ELSE'), ("right", '='), ("left", 'OR'), ("left", 'AND'), ("left", '|'), ("left", '^'), ("left", '&'), ("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'), ("left", 'SHL', 'SHR'), ("left", '+', '-'), ("left", '*', '/', '%'), ) def p_error(self, p): if p: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')".format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) else: print("Unexpected end of input") def p_program(self, p): """program : podprogramy""" p[0] = AST.Program(p[1]) def p_podprogramy(self, p): """podprogramy : podprogramy podprogram | """ if len(p) == 3: podprogram = p[2] if p[1] == None: p[0] = AST.Podprogramy() else: p[0] = p[1] p[0].addPodprogram(podprogram) else: p[0] = AST.Podprogramy() def p_podprogram(self, p): """podprogram : declaration | fundef | instruction | """ p[0] = AST.Podprogram(p[1]) def p_declaration(self, p): """declaration : TYPE inits ';' | error ';' """ if len(p) == 4: type = p[1] inits = p[2] p[0] = AST.Declaration(type, inits) else: p[0] = AST.Error(p[1]) def p_inits(self, p): """inits : inits ',' init | init """ if len(p) == 4: if p[1] == None: p[0] = AST.InitList(p[3]) else: p[0] = p[1] p[0].addInit(p[3]) else: p[0] = AST.InitList(p[1]) def p_init(self, p): """init : ID '=' expression """ id = p[1] exp = p[3] p[0] = AST.Init(id, exp, p.lineno(1)) def p_instructions(self, p): """instructions : instruction instructions | instruction """ if len(p) == 3: if p[2] == None: p[0] = AST.InstList(p[1]) else: p[0] = p[2] p[0].addInst(p[1]) else: p[0] = AST.InstList(p[1]) def p_instruction(self, p): """instruction : print_instr | labeled_instr | assignment | choice_instr | while_instr | repeat_instr | return_instr | break_instr | continue_instr | compound_instr | expression ';' """ p[0] = p[1] def p_print_instr(self, p): """print_instr : PRINT expr_list ';' | PRINT error ';' """ data = p[2] p[0] = AST.PrtInst(data, p.lineno(1)) def p_labeled_instr(self, p): """labeled_instr : ID ':' instruction """ id = p[1] inst = p[3] p[0] = AST.LabInst(id, inst, p.lineno(1)) def p_assignment(self, p): """assignment : ID '=' expression ';' """ id = p[1] exp = p[3] p[0] = AST.AsgInst(id,exp, p.lineno(1)) def p_choice_instr(self, p): """choice_instr : IF '(' condition ')' instruction %prec IFX | IF '(' condition ')' instruction ELSE instruction | IF '(' error ')' instruction %prec IFX | IF '(' error ')' instruction ELSE instruction """ cond = p[3] inst = p[5] if len(p) >= 8: alt = p[7] else: alt = None p[0] = AST.ChoiInst(cond,inst,alt) def p_while_instr(self, p): """while_instr : WHILE '(' condition ')' instruction | WHILE '(' error ')' instruction """ cond = p[3] inst = p[5] p[0] = AST.WhiInst(cond, inst) def p_repeat_instr(self, p): """repeat_instr : REPEAT instructions UNTIL condition ';' """ inst = p[2] cond = p[4] p[0] = AST.RepInst(inst,cond) def p_return_instr(self, p): """return_instr : RETURN expression ';' """ exp = p[2] p[0] = AST.RetInst(exp, p.lineno(1)) def p_continue_instr(self, p): """continue_instr : CONTINUE ';' """ p[0] = AST.Continue(p.lineno(1)) def p_break_instr(self, p): """break_instr : BREAK ';' """ p[0] = AST.Break(p.lineno(1)) #Przypilnowac by instruction nie zawieral fundefu #stworzyc unit -> declaration | instruction #i jeszcze stworzyc kolejna co by miala liste unitow. #poczatek zmian def p_units(self, p): """units : units unit | """ if len(p) == 3: unit = p[2] if p[1] == None: p[0] = AST.Units() else: p[0] = p[1] p[0].addUnit(unit) else: p[0] = AST.Units() def p_unit(self, p): """unit : declaration | instruction | """ p[0] = AST.Unit(p[1]); #koniec zmian def p_compound_instr(self, p): """compound_instr : '{' units '}' """ if len(p[2].children) == 0: p[0] = AST.CompInst(None, p.lineno(3)) else: p[0] = AST.CompInst(p[2], p.lineno(3)) def p_condition(self, p): """condition : expression""" p[0] = p[1] def p_const(self, p): """const : integer | float | string""" p[0] = p[1] def p_integer(self, p): """integer : INTEGER""" p[0] = AST.Integer(p[1], p.lineno(1)) def p_float(self, p): """float : FLOAT""" p[0] = AST.Float(p[1], p.lineno(1)) def p_string(self, p): """string : STRING""" text = filter(lambda c: c not in "\"", p[1]) p[0] = AST.String(text, p.lineno(1)) def p_expression_id(self, p): """expression : ID""" p[0] = AST.Variable(p.lineno(1), p[1]) #H4X10R5K1 sposob na odronienie ID od const def p_expression(self, p): """expression : const | expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression '%' expression | expression '|' expression | expression '&' expression | expression '^' expression | expression AND expression | expression OR expression | expression SHL expression | expression SHR expression | expression EQ expression | expression NEQ expression | expression '>' expression | expression '<' expression | expression LE expression | expression GE expression | '(' expression ')' | '(' error ')' | ID '(' expr_list_or_empty ')' | ID '(' error ')' """ if len(p) == 2: p[0] = p[1] elif p[2] == '(' and p[1] != '(': function_name = p[1] parameters = p[3] p[0] = AST.CallExp(function_name, parameters, p.lineno(1)) elif p[1] == '(': p[0] = p[2] else: left = p[1] operator = p[2] right = p[3] p[0] = AST.BinExpr(left, operator, right, p.lineno(2)) def p_expr_list_or_empty(self, p): """expr_list_or_empty : expr_list | """ if len(p) == 2: p[0] = p[1] else: p[0] = None def p_expr_list(self, p): """expr_list : expr_list ',' expression | expression """ if len(p) == 4: if p[1] == None: p[0] = AST.ExpList(p[3]) else: p[0] = p[1] p[0].addExp(p[3]) else: p[0] = AST.ExpList(p[1]) def p_fundef(self, p): """fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """ type = p[1] name = p[2] parameters = p[4] body = p[6] p[0] = AST.Function(type, name, parameters, body, p.lineno(1)) def p_args_list_or_empty(self, p): """args_list_or_empty : args_list | """ if len(p) == 2: p[0] = p[1] else: p[0] = None def p_args_list(self, p): """args_list : args_list ',' arg | arg """ if len(p) == 4: if p[3] == None: p[0] = AST.ArgList(p[3]) else: p[0] = p[1] p[0].addArgToList(p[3]) else: p[0] = AST.ArgList(p[1]) def p_arg(self, p): """arg : TYPE ID """ type = p[1] name = p[2] p[0] = AST.Parameter(type, name, p.lineno(1))
def new_scanner(): '''returns an instance of class Scanner''' new_names = Names() new_scanner = Scanner('test_files/ripple_counter.txt', new_names) return new_scanner
def format(self, sourceStr): normalizedStr = "" scanner = Scanner(sourceStr); while scanner.hasNext(): normalizedStr += str(scanner.next()) + " " return normalizedStr;
class Parser(object): def parse(self, sourceStr): self._completionMessage = "No errors" self._parseSuccessful = True self._scanner = Scanner(sourceStr) self._expression() self._accept(self._scanner.get(), Token.EOE, "symbol after end of expression") def parseStatus(self): return self._completionMessage def _accept(self, token, expected, errorMessage): if token.getType() != expected: self._fatalError(token, errorMessage) def _fatalError(self, token, errorMessage): self._parseSuccessful = False self._completionMessage = "Parsing error -- " + \ errorMessage + \ "\nExpression so far = " + \ self._scanner.stringUpToCurrentToken() raise Exception, self._completionMessage def _expression(self): """Syntax rule: expression = term { addingOperator term } """ self._term() token = self._scanner.get() while token.getType() in (Token.PLUS, Token.MINUS): self._scanner.next() self._term() token = self._scanner.get() def _term(self): """Syntax rule: term = factor { multiplyingOperator factor } """ self._factor() token = self._scanner.get() while token.getType() in (Token.MUL, Token.DIV): self._scanner.next() self._factor() token = self._scanner.get() def _factor(self): """Syntax rule: factor = number | "(" expression ") """" token = self._scanner.get() if token.getType() == Token.INT: self._scanner.next() elif token.getType() == Token.L_PAR: self._scanner.next() self._expression() self._accept(self._scanner.get(), Token.R_PAR, "')' expected") self._scanner.next() else: self._fatalError(token, "bad factor")
print "buckets path:", buckets_dir with open(index_path) as f: reader = csv.reader(f) items = list(reader) if sum(map(bool, (args.scanner, args.max_order, args.word))) > 1: parser.error( 'can only specify one of --word, --scanner and --max_order') # Tokenize if not SILENT: print "will tokenize %d files" % len(items) if args.scanner: from scanner import Scanner tokenizer = Scanner.from_file(args.scanner) if not SILENT: print "using provided scanner: ", args.scanner elif args.word: tokenizer = str.split if not SILENT: print "using str.split to tokenize" else: min_order = args.min_order if args.min_order else MIN_NGRAM_ORDER max_order = args.max_order if args.max_order else MAX_NGRAM_ORDER tokenizer = NGramTokenizer(min_order, max_order) if not SILENT: print "using n-gram tokenizer: min_order({0}) max_order({1})".format( min_order, max_order) if args.term_freq: if not SILENT:
} if char not in "".join(Token.get_operators()): self.error("Unknown operator starting with '{}'".format(char)) next_char = self.scanner.get_next_char() possible_op = char + next_char.char # try to create a two-char operator; does it exist? if (possible_op) in d.keys(): return Token(d[possible_op], possible_op) # it is a one-char operator; store the 2nd char else: self.current_char = next_char self.current_char_consumed = False return Token(d[char], char) def error(self, msg): """Raise an error with a custom message""" raise Exception(msg) if __name__ == "__main__": while True: text = input(">> ") scan = Scanner(text) Tok = Tokenizer(scan) token = Tok.get_next_token() print(token) while token.get_type() is not Token.EOF: token = Tok.get_next_token() print(token)
def generate_xml(self, directory): __scanner = Scanner(directory) __scanner.generate_xml()
class Cparser(object): def __init__(self): self.scanner = Scanner() self.scanner.build() self.errorsOccured = False tokens = Scanner.tokens precedence = ( ("nonassoc", 'IFX'), ("nonassoc", 'ELSE'), ("right", '='), ("left", 'OR'), ("left", 'AND'), ("left", '|'), ("left", '^'), ("left", '&'), ("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'), ("left", 'SHL', 'SHR'), ("left", '+', '-'), ("left", '*', '/', '%'), ) def p_error(self, p): if p: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')". format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) else: print('At end of input') self.errorsOccured = True def p_program(self, p): """program : classdefs declarations fundefs instructions""" p[0] = AST.Program(p[1], p[2], p[3], p[4]) def p_declarations(self, p): """declarations : declarations declaration | """ if len(p) > 1: p[1].list.append(p[2]) p[0] = p[1] else: p[0] = AST.Declarations() def p_declaration(self, p): """declaration : TYPE inits ';' | id classinits ';' | error ';' """ if len(p) > 2: p[0] = AST.Declaration(p[1], p[2]) else: p[0] = p[1] def p_inits(self, p): """inits : inits ',' init | init """ if len(p) > 2: p[1].list.append(p[3]) p[0] = p[1] else: inits = AST.Inits() inits.list.append(p[1]) p[0] = inits def p_init(self, p): """init : id '=' expression """ p[0] = AST.Init(p[1], p[3]) def p_classinits(self, p): """classinits : classinits ',' classinit | classinit """ if len(p) > 2: p[1].list.append(p[3]) p[0] = p[1] else: classinits = AST.Classinits() classinits.list.append(p[1]) p[0] = classinits def p_classinit(self, p): """classinit : id """ p[0] = AST.Classinit(p[1]) def p_instructions(self, p): """instructions : instructions instruction | instruction """ if len(p) == 2: instructions = AST.Instructions() instructions.list.append(p[1]) p[0] = instructions else: p[1].list.append(p[2]) p[0] = p[1] def p_instruction(self, p): """instruction : expression ';' | print_instr | labeled_instr | assignment | choice_instr | while_instr | repeat_instr | return_instr | break_instr | continue_instr | compound_instr""" p[0] = p[1] def p_print_instr(self, p): """print_instr : PRINT expression ';' | PRINT error ';' """ p[0] = AST.PrintInstr(p[2]) def p_labeled_instr(self, p): """labeled_instr : id ':' instruction """ p[0] = AST.LabeledInstr(p[1], p[3]) def p_assignment(self, p): """assignment : access '=' expression ';' """ p[0] = AST.Assignment(p[1], p[3]) def p_access(self, p): """access : id | id '.' id """ if len(p) == 2: access = AST.Access(p.lineno(1)) access.list.append(p[1]) p[0] = access else: access = AST.Access(p.lineno(1)) access.list.append(p[1]) access.list.append(p[3]) p[0] = access def p_choice_instr(self, p): """choice_instr : IF '(' condition ')' instruction %prec IFX | IF '(' condition ')' instruction ELSE instruction | IF '(' error ')' instruction %prec IFX | IF '(' error ')' instruction ELSE instruction """ if len(p) == 6: p[0] = AST.ChoiceInstr(p[3], p[5], None) else: p[0] = AST.ChoiceInstr(p[3], p[5], p[7]) def p_while_instr(self, p): """while_instr : WHILE '(' condition ')' instruction | WHILE '(' error ')' instruction """ p[0] = AST.WhileInstr(p[3], p[5]) def p_repeat_instr(self, p): """repeat_instr : REPEAT instructions UNTIL condition ';' """ p[0] = AST.RepeatInstr(p[2], p[4]) def p_return_instr(self, p): """return_instr : RETURN expression ';' """ p[0] = AST.ReturnInstr(p[2]) def p_continue_instr(self, p): """continue_instr : CONTINUE ';' """ p[0] = AST.Continue() def p_break_instr(self, p): """break_instr : BREAK ';' """ p[0] = AST.Break() def p_compound_instr(self, p): """compound_instr : '{' declarations instructions '}'""" p[0] = AST.CompoundInstr(p[2], p[3]) def p_condition(self, p): """condition : expression""" p[0] = AST.Condition(p[1]) def p_const(self, p): """const : integer | float | string""" p[0] = p[1] def p_integer(self, p): """integer : INTEGER""" p[0] = AST.Integer(p[1], p.lineno(1)) def p_float(self, p): """float : FLOAT""" p[0] = AST.Float(p[1], p.lineno(1)) def p_string(self, p): """string : STRING""" p[0] = AST.String(p[1][1:-1], p.lineno(1)) def p_id(self, p): """id : ID""" p[0] = AST.Id(p[1], p.lineno(1)) def p_expression(self, p): """expression : const | access | expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression '%' expression | expression '|' expression | expression '&' expression | expression '^' expression | expression AND expression | expression OR expression | expression SHL expression | expression SHR expression | expression EQ expression | expression NEQ expression | expression '>' expression | expression '<' expression | expression LE expression | expression GE expression | '(' expression ')' | '(' error ')' | access '(' expr_list_or_empty ')' | access '(' error ')' """ if len(p) == 2: p[0] = p[1] elif p[1] == '(': p[0] = AST.ParExpr(p[2]) elif len(p) == 4: p[0] = AST.BinExpr(p[1], p[2], p[3]) else: p[0] = AST.FunExpr(p[1], p[3]) def p_expr_list_or_empty(self, p): """expr_list_or_empty : expr_list | """ if len(p) == 1: p[0] = AST.ExprList() else: p[0] = p[1] def p_expr_list(self, p): """expr_list : expr_list ',' expression | expression """ if len(p) == 4: p[1].list.append(p[3]) p[0] = p[1] else: toReturn = AST.ExprList() toReturn.list.append(p[1]) p[0] = toReturn def p_fundefs(self, p): """fundefs : fundef fundefs | """ if len(p) == 3: p[2].list.reverse() p[2].list.append(p[1]) p[2].list.reverse() p[0] = p[2] else: p[0] = AST.FunDefs() def p_fundef(self, p): """fundef : TYPE id '(' args_list_or_empty ')' compound_instr | id id '(' args_list_or_empty ')' compound_instr""" p[0] = AST.FunDef(p[1], p[2], p[4], p[6]) def p_args_list_or_empty(self, p): """args_list_or_empty : args_list | """ if len(p) == 1: p[0] = AST.ArgList() else: p[0] = p[1] def p_args_list(self, p): """args_list : args_list ',' arg | arg """ if len(p) == 2: argList = AST.ArgList() argList.list.append(p[1]) p[0] = argList else: p[1].list.append(p[3]) p[0] = p[1] def p_arg(self, p): """arg : TYPE id | id id""" p[0] = AST.Arg(p[1], p[2]) def p_classdefs(self, p): """classdefs : classdef classdefs | """ if len(p) == 3: p[2].list.reverse() p[2].list.append(p[1]) p[2].list.reverse() p[0] = p[2] else: p[0] = AST.ClassDefs() def p_classdef(self, p): """classdef : accessmodificator CLASS id classcontent | accessmodificator CLASS id EXTENDS id classcontent""" if len(p) < 7: p[0] = AST.ClassDef(p[1], p[3], None, p[4]) else: p[0] = AST.ClassDef(p[1], p[3], p[5], p[6]) def p_classcontent(self, p): """classcontent : '{' fielddefs ';' methoddefs '}' """ p[0] = AST.Classcontent(p[2], p[4]) def p_fielddefs(self, p): """fielddefs : fielddef fielddefs | """ if len(p) == 3: p[2].list.reverse() p[2].list.append(p[1]) p[2].list.reverse() p[0] = p[2] else: p[0] = AST.Fielddefs() def p_fielddef(self, p): """fielddef : accessmodificator declaration """ p[0] = AST.Fielddef(p[1], p[2]) def p_methoddefs(self, p): """methoddefs : methoddef methoddefs | """ if len(p) == 3: p[2].list.reverse() p[2].list.append(p[1]) p[2].list.reverse() p[0] = p[2] else: p[0] = AST.Methoddefs() def p_methoddef(self, p): """methoddef : accessmodificator fundef""" p[0] = AST.Methoddef(p[1], p[2]) def p_accessmodificator(self, p): """accessmodificator : PRIVATE | PROTECTED | PUBLIC""" p[0] = p[1]
def test_open_file_(names): """Test file open within scanner.py""" f = open('test_specfiles/test_scanner/test_open_file.txt') scanner = Scanner('test_specfiles/test_scanner/test_open_file.txt', names) f1 = scanner._open_file(scanner.path) assert f.read() == f1.read()
class Cparser(object): def __init__(self): self.scanner = Scanner() self.scanner.build() tokens = Scanner.tokens precedence = ( ("nonassoc", 'IFX'), ("nonassoc", 'ELSE'), ("right", '='), ("left", 'OR'), ("left", 'AND'), ("left", '|'), ("left", '^'), ("left", '&'), ("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'), ("left", 'SHL', 'SHR'), ("left", '+', '-'), ("left", '*', '/', '%'), ) def p_error(self, p): if p: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')". format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) else: print("Unexpected end of input") def p_program(self, p): """program : blocks""" p[0] = AST.Program(p[1]) TreePrinter() print p[0] def p_blocks(self, p): """blocks : blocks block | """ if len(p) == 3: p[0] = p[1] + [p[2]] else: p[0] = [] def p_block(self, p): """block : declaration | instruction | fundef""" p[0] = AST.Block(p[1]) def p_declaration(self, p): """declaration : TYPE inits ';' | error ';' """ if len(p) > 3: p[0] = AST.Declaration(p[2]) def p_inits(self, p): """inits : inits ',' init | init """ if len(p) == 4: p[0] = p[1] + [p[3]] else: p[0] = [p[1]] def p_init(self, p): """init : ID '=' expression """ p[0] = AST.Variable(p[1], p[3]) def p_instruction(self, p): """instruction : print_instr | labeled_instr | assignment | choice_instr | while_instr | repeat_instr | return_instr | break_instr | continue_instr | compound_instr | expression ';' """ if len(p) == 2: p[0] = p[1] else: p[0] = [p[1]] def p_print_instr(self, p): """print_instr : PRINT expr_list ';' | PRINT error ';' """ p[0] = [AST.Instruction(p[1], p[2])] def p_labeled_instr(self, p): """labeled_instr : ID ':' instruction """ p[0] = [AST.Instruction(p[2], [p[1], p[3]])] def p_assignment(self, p): """assignment : ID '=' expression ';' """ p[0] = [AST.Instruction(p[2], [p[1], p[3]])] def p_choice_instr(self, p): """choice_instr : IF '(' condition ')' instruction %prec IFX | IF '(' condition ')' instruction ELSE instruction | IF '(' error ')' instruction %prec IFX | IF '(' error ')' instruction ELSE instruction """ if len(p) == 6: p[0] = [AST.Instruction(p[1], [p[3], p[5]]) ] #Instruction(IF, [condition, instruction]) elif len(p) == 8: p[0] = [ AST.Instruction(p[1], [p[3], p[5]]), AST.Instruction(p[6], [p[7]]) ] #Instruction(IF, [condition, instruction]) #Instruction(ELSE, [instruction]) def p_while_instr(self, p): """while_instr : WHILE '(' condition ')' instruction | WHILE '(' error ')' instruction """ p[0] = [AST.Instruction(p[1], [p[3], p[5]])] def p_repeat_instr(self, p): """repeat_instr : REPEAT instruction UNTIL condition ';' """ p[0] = [AST.Instruction(p[1], [p[2]]), AST.Instruction(p[3], [p[4]])] def p_return_instr(self, p): """return_instr : RETURN expression ';' """ p[0] = [AST.Instruction(p[1], [p[2]])] def p_continue_instr(self, p): """continue_instr : CONTINUE ';' """ p[0] = [AST.Instruction(p[1], [])] def p_break_instr(self, p): """break_instr : BREAK ';' """ p[0] = [AST.Instruction(p[1], [])] def p_compound_instr(self, p): """compound_instr : '{' comp_instr_blocks '}' """ p[0] = p[2] def p_comp_instr_blocks(self, p): """comp_instr_blocks : comp_instr_blocks comp_instr_block | """ if len(p) == 3: p[0] = p[1] + [p[2]] else: p[0] = [] def p_comp_instr_block(self, p): """comp_instr_block : declaration | instruction""" p[0] = AST.Block(p[1]) def p_condition(self, p): """condition : expression""" p[0] = p[1] def p_const(self, p): """const : INTEGER | FLOAT | STRING""" if type(p[1]) == int: p[0] = AST.Const(AST.Integer(p[1])) elif type(p[1]) == float: p[0] = AST.Const(AST.Float(p[1])) elif type(p[1]) == str: p[0] = AST.Const(AST.String(p[1])) def p_expression(self, p): """expression : const | ID | expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression '%' expression | expression '|' expression | expression '&' expression | expression '^' expression | expression AND expression | expression OR expression | expression SHL expression | expression SHR expression | expression EQ expression | expression NEQ expression | expression '>' expression | expression '<' expression | expression LE expression | expression GE expression | '(' expression ')' | '(' error ')' | ID '(' expr_list_or_empty ')' | ID '(' error ')' """ if len(p) == 2: p[0] = p[1] elif len(p) == 4: if (p[1] == '('): p[0] = p[2] else: p[0] = AST.BinExpr(p[2], p[1], p[3]) elif len(p) == 5: p[0] = AST.Instruction('FUNCALL', [p[1], p[3]]) def p_expr_list_or_empty(self, p): """expr_list_or_empty : expr_list | """ if len(p) == 2: p[0] = p[1] else: p[0] = [] def p_expr_list(self, p): """expr_list : expr_list ',' expression | expression """ if len(p) == 4: p[0] = p[1] + [p[3]] else: p[0] = [p[1]] def p_fundef(self, p): """fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """ p[0] = AST.Fundef(p[1], p[2], p[4], p[6]) def p_args_list_or_empty(self, p): """args_list_or_empty : args_list | """ if len(p) == 2: p[0] = p[1] else: p[0] = [] def p_args_list(self, p): """args_list : args_list ',' arg | arg """ if len(p) == 4: p[0] = p[1] + [p[3]] else: p[0] = [p[1]] def p_arg(self, p): """arg : TYPE ID """ p[0] = AST.Arg(p[1], p[2])
def scan(self, path): with Scanner(path) as sc: sc.go()
class Cparser(object): def __init__(self): self.error_occured = False self.scanner = Scanner() self.scanner.build() tokens = Scanner.tokens precedence = ( ("nonassoc", 'IFX'), ("nonassoc", 'ELSE'), ("right", '='), ("left", 'OR'), ("left", 'AND'), ("left", '|'), ("left", '^'), ("left", '&'), ("nonassoc", '<', '>', 'EQ', 'NEQ', 'LE', 'GE'), ("left", 'SHL', 'SHR'), ("left", '+', '-'), ("left", '*', '/', '%'), ) def p_error(self, p): if p: print("Syntax error at line {0}, column {1}: LexToken({2}, '{3}')". format(p.lineno, self.scanner.find_tok_column(p), p.type, p.value)) else: print("Unexpected end of input") def p_program(self, p): """program : main_parts_interlacing""" #p[0] = ('program',p[1],p[2],p[3]) # print "{0}".format(p.lineno(1)) p[0] = Program(p[1]) def p_main_parts_interlacing(self, p): """main_parts_interlacing : main_parts_interlacing main_part | main_part """ if p.__len__() == 3: p[0] = MainPartsInerlacing(p[1], p[2]) else: p[0] = MainPartsInerlacing(None, p[1]) def p_main_part(self, p): """ main_part : declaration | fundef | instruction """ p[0] = p[1] def p_declarations(self, p): """declarations : declarations declaration | """ #p[0] = ('') if p.__len__() == 1 else ('decl',p[1], p[2]) p[0] = None if p.__len__() == 1 else Declarations(p[1], p[2]) def p_declaration(self, p): """declaration : TYPE inits ';' | error ';' """ #p[0] = ('declaration',p[1], p[2]) if p.__len__() == 4 else ('declaration-error',p[1]) #p[0] = Declaration(p[1], p[2]) if not type(p[1]) == LexToken: p[0] = Declaration(p[1], p[2]) else: self.error_occured = True def p_inits(self, p): """inits : inits ',' init | init """ #p[0] = ('inits',p[1],p[3]) if p.__len__() == 4 \ # else ('init',p[1]) p[0] = Inits(p[1],p[3]) if p.__len__() == 4 \ else p[1] def p_init(self, p): """init : ID '=' expression """ #p[0] = ('init','=',p[1],p[3]) p[0] = Init(p[1], p[3]) def p_instructions_opt(self, p): """instructions_opt : instructions | """ #p[0] = ('instructions_opt',p[1]) if p.__len__()==2 else('empty_instr') p[0] = InstructionsOpt(p[1]) if p.__len__() == 2 else None def p_instructions(self, p): """instructions : instructions instruction | instruction """ #p[0] = ('instructions',p[1],p[2]) if p.__len__()==3 else ('instruction', p[1]) p[0] = Instructions(p[1], p[2]) if p.__len__() == 3 else p[1] def p_instruction(self, p): """instruction : print_instr | labeled_instr | assignment | choice_instr | while_instr | repeat_instr | return_instr | break_instr | continue_instr | compound_instr | expression ';' """ #p[0] = ('instruction-node',p[1]) p[0] = p[1] def p_print_instr(self, p): """print_instr : PRINT expr_list ';' | PRINT error ';' """ #p[0] = ('print_instr', p[1], p[2]) if not type(p[2]) == LexToken: p[0] = PrintInstr(p[2]) else: self.error_occured = True def p_labeled_instr(self, p): """labeled_instr : ID ':' instruction """ #p[0] = ('labeled_instr', p[1], p[3]) p[0] = LabeledInstr(p[1], p[3]) def p_assignment(self, p): """assignment : ID '=' expression ';' """ #p[0] = ('assignment',p[1],p[3]) p[0] = Assignment(p[1], p[3]) def p_choice_instr(self, p): """choice_instr : IF '(' condition ')' instruction %prec IFX | IF '(' condition ')' instruction ELSE instruction | IF '(' error ')' instruction %prec IFX | IF '(' error ')' instruction ELSE instruction """ #p[0] = ('if-cond-instr',p[3],p[5])if p.__len__() == 6 else ('if-cond-instr-else-instr',p[3],p[5],p[7]) if not type(p[3]) == LexToken: p[0] = ChoiceInstr(p[3], p[5]) if p.__len__() == 6 else ChoiceInstr( p[3], p[5], p[7]) else: self.error_occured = True def p_while_instr(self, p): """while_instr : WHILE '(' condition ')' instruction | WHILE '(' error ')' instruction """ #p[0] = ('while-instr',p[3],p[5]) if not type(p[3]) == LexToken: p[0] = WhileInstr(p[3], p[5]) else: self.error_occured = True def p_repeat_instr(self, p): """repeat_instr : REPEAT instructions UNTIL condition ';' """ #p[0] = ('repeat',p[2],p[4]) p[0] = RepeatInstr(p[2], p[4]) def p_return_instr(self, p): """return_instr : RETURN expression ';' """ #p[0] = ('return',p[2]) p[0] = ReturnInstr(p[2], p[1].lineno, p[1].column) def p_continue_instr(self, p): """continue_instr : CONTINUE ';' """ #p[0] = ('continue') p[0] = ContinueInstr(p[1].lineno, p[1].column) def p_break_instr(self, p): """break_instr : BREAK ';' """ #p[0] = ('break') p[0] = BreakInstr(p[1].lineno, p[1].column) def p_compound_instr(self, p): """compound_instr : '{' declarations instructions_opt '}' """ #p[0] = ('compound_instr',p[2],p[3]) p[0] = CompoundInstr(p[2], p[3]) def p_condition(self, p): """condition : expression""" #p[0] = ('condition',p[1]) p[0] = Condition(p[1]) def p_const(self, p): """const : INTEGER | FLOAT | STRING""" #p[0] = p[1] #p[0] = Const(p[1]) #to-do!! p[0] = p[1] def p_expression(self, p): """expression : const | ID | expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression '%' expression | expression '|' expression | expression '&' expression | expression '^' expression | expression AND expression | expression OR expression | expression SHL expression | expression SHR expression | expression EQ expression | expression NEQ expression | expression '>' expression | expression '<' expression | expression LE expression | expression GE expression | '(' expression ')' | '(' error ')' | ID '(' expr_list_or_empty ')' | ID '(' error ')' """ #p[0] = (p[1]) if p.__len__() == 2 else \ # (p[2],p[1],p[3]) if p.__len__() == 4 and p[1]!='(' else \ # p[2] if p.__len__() == 4 and p[1]=='(' else (p[1],p[3]) """ p[0] = p[1] if p.__len__() == 2 else \ BinExpr(p[2],p[1],p[3]) if p.__len__() == 4 and p[1]!='(' else \ p[2] if p.__len__() == 4 and p[1]=='(' else DeclaredFunc(p[1],p[3]) """ if p.__len__() == 2: p[0] = p[1] elif p.__len__() == 4: if p[1] != '(': p[0] = BinExpr(p[2], p[1], p[3]) else: if not type(p[2]) == LexToken: p[0] = p[2] else: self.error_occured = True else: if not type(p[3]) == LexToken: p[0] = DeclaredFunc(p[1], p[3]) else: self.error_occured = True def p_expr_list_or_empty(self, p): """expr_list_or_empty : expr_list | """ #p[0] = ('expr_list',p[1]) if p.__len__()==2 else ('empty_exprlist') p[0] = PExprListOrEmpty(p[1]) if p.__len__() == 2 else None def p_expr_list(self, p): """expr_list : expr_list ',' expression | expression """ #p[0] = ('expr_list',p[1],p[3]) if p.__len__() == 4 else ('expr',p[1]) p[0] = PExprList(p[1], p[3]) if p.__len__() == 4 else p[1] def p_fundefs_opt(self, p): """fundefs_opt : fundefs | """ #p[0] = ('fundefs_opt',p[1]) if p.__len__() == 2 else ('no fundefs') p[0] = FuncDefsOpt(p[1]) if p.__len__() == 2 else None def p_fundefs(self, p): """fundefs : fundefs fundef | fundef """ #p[0] = ('fundefs',p[1],p[2]) if p.__len__() == 3 else ("fundef", p[1]) p[0] = FuncDefs(p[1], p[2]) if p.__len__() == 3 else p[1] def p_fundef(self, p): """fundef : TYPE ID '(' args_list_or_empty ')' compound_instr """ #p[0] = ('fundef',p[2],p[1],p[4],p[6]) p[0] = FuncDef(p[1], p[2], p[4], p[6]) def p_args_list_or_empty(self, p): """args_list_or_empty : args_list | """ #p[0] = ('args_list_',p[1]) if p.__len__() == 2 else ("empty-args-list") p[0] = p[1] if p.__len__() == 2 else None def p_args_list(self, p): """args_list : args_list ',' arg | arg """ #p[0] = ('args_list',p[1],p[3]) if p.__len__() == 4 else ('arg', p[1]) p[0] = ArgsList(p[1], p[3]) if p.__len__() == 4 else p[1] def p_arg(self, p): """arg : TYPE ID """ #p[0] = ('arg',p[1],p[2]) p[0] = Arg(p[1], p[2])
# !/bin/python import sys import ply.lex as lex from scanner import Scanner if __name__ == '__main__': try: filename = sys.argv[1] if len( sys.argv) > 1 else "examples/scannerexample" file = open(filename, "r") except IOError: print("Cannot open {0} file".format(filename)) sys.exit(0) text = file.read() scanner = Scanner() lexer = scanner.lexer lexer.input(text) # Give the lexer some input # Tokenize while True: tok = lexer.token() if not tok: break # No more input column = scanner.find_column(tok) print("(%d,%d): %s(%s)" % (tok.lineno, column, tok.type, tok.value))
from scanner import Scanner scanner = Scanner('input.in', 'parser.automata') ''' stringurile merg pe mai multe linii doar daca are \ ''' while True: token = scanner.get_next_token() if token is None: break print(token)
def __init__(self): self.scanner = Scanner() self.scanner.build() self.args = {} self.const = {}
import re import os import json from scanner import Scanner revp_ptn = re.compile(r'(\d+)_(\d+)') revp_list = {} data = {} ret_ex = [] ret_vac = [] path = 'D:\github\data\shop_review' uid_ptn = re.compile(r'href="/member/(\d+)(?:\?[^"]+)?"') filename = 'uid.json' s = Scanner(filename, uid_ptn) # 找到每个店铺的评价页面的用户id,输出在uid.json中,key为店铺评价页面,values为对应页面的 # 评价的用户id s.scan(path, save_period=500) count = 0 rid = [] for k, v in s.data.items(): # 店铺评价页面的首页没有评价的店铺。输出在 rid.json中。 if len(v) == 0 and k.endswith('_1.html'): count += 1 rid.append(k) print count # 输出符合条件的店铺总数 with open("rid.json", 'w ') as fp: json.dump(rid, fp) for f in os.listdir(path): # 搜索目标文件夹中的文件名(店铺及评价页面),输出在revp.json中
class Parser(object): def parse(self, sourceStr): self._completionMessage = "No errors" self._parseSuccessful = True self._scanner = Scanner(sourceStr) tree = self._expression() self._accept(self._scanner.get(), Token.EOE, "symbol after end of expression") return tree def parseStatus(self): return self._completionMessage def _accept(self, token, expected, errorMessage): if token.getType() != expected: self._fatalError(token, errorMessage) def _fatalError(self, token, errorMessage): self._parseSuccessful = False self._completionMessage = "Parsing error -- " + \ errorMessage + \ "\nExpression so far = " + \ self._scanner.stringUpToCurrentToken() raise Exception(self._completionMessage) def _expression(self): tree = self._term() token = self._scanner.get() while token.getType() in (Token.PLUS, Token.MINUS): op = str(token) self._scanner.next() tree = InteriorNode(op, tree, self._term()) token = self._scanner.get() return tree def _term(self): tree = self._factor() token = self._scanner.get() while token.getType() in (Token.MUL, Token.DIV): op = str(token) self._scanner.next() tree = InteriorNode(op, tree, self._factor()) token = self._scanner.get() return tree def _factor(self): tree = self._primary() token = self._scanner.get() if token.getType() == Token.EXPO: op = str(token) self._scanner.next() tree = InteriorNode(op, tree, self._factor()) token = self._scanner.get() return tree def _primary(self): token = self._scanner.get() if token.getType() == Token.INT: tree = LeafNode(token.getValue()) self._scanner.next() elif token.getType() == Token.L_PAR: self._scanner.next() tree = self._expression() self._accept(self._scanner.get(), Token.R_PAR, "')' expected") self._scanner.next() else: tree = None self._fatalError(token, "bad primary") return tree
import click from scanner import Scanner from table import Table import os import time from progress.bar import ChargingBar table = Table() scan = Scanner() class Functions(): def __init__(self): pass @click.command() @click.option('--nodes') def view_status(nodes): while True: data = scan.run_scan(nodes) os.system('cls') tbl = table.render_table( data, headers=["Address", "Status", "Time (ms)"]) print(tbl) @click.command() @click.option('--nodes') def ip_scan(nodes): data = scan.get_range(nodes) bar = ChargingBar('Scanning', max=len(data)) discovered = [] for ip in data:
from scanner import Scanner scanner = Scanner() data = {} import csv verses = {} multiple_matches = [] total_verses = 0 total_matches = 0 with open('data/verses.csv', 'rb') as csvfile: versereader = csv.reader(csvfile, delimiter=',', quotechar='|') count = 0 multiple_match_count = 0 for row in versereader: (verse_id, input_string, real_scan) = row scan = scanner.scan(input_string, known_only = True) # if not(len(scan['results'])>0): # # print 'Error '+verse_id+":"+input_string # scan = scanner.scan(input_string, known_only = False) # print scanner.print_scan(scan) # scan = scanner.scan(input_string, known_only = True) total_matches+=len(scan['results']) total_verses +=1 assert len(scan['results'])>0 print 'total verses = '+str(total_verses) print 'total matches = '+str(total_matches) assert total_verses==3314