def diff(source1, source2, dest, config): IDS = config["IDS"] REQUIRED_ATTRS = config["REQUIRED_ATTRS"] (fname1, need_cleanup1) = process_source(source1) (fname2, need_cleanup2) = process_source(source2) try: p = parser.Parser() parsed1 = p.parse_file(open(fname1)) p = parser.Parser() parsed2 = p.parse_file(open(fname2)) dest = open(dest, "w") diffed = parsed1.diff(parsed2, path="", ids=IDS, required=REQUIRED_ATTRS) del parsed1, parsed2 writer = xmlwriter.XMLWriter(dest, encoding="utf-8") writer.start_document() saxxml.diff_tree2xml(writer, diffed, required_attrs=REQUIRED_ATTRS) writer.end_document() except Exception: raise finally: if need_cleanup1: os.remove(fname1) if need_cleanup2: os.remove(fname2)
def loadFiles(fnameList): mainParser = None with open(fnameList[0], 'rb') as f: mainParser = parser.Parser(f.read()) if mainParser.smsXML: collection = mainParser.soup.find('smses') else: collection = mainParser.soup.find('calls') if len(fnameList) > 1: for i in range(1, len(fnameList)): with open(fnameList[i], 'rb') as f: mergeParser = parser.Parser(f.read()) if mergeParser.smsXML != mainParser.smsXML: raise Exception('cannot merge mixed file types') if mainParser.smsXML: nodes = mergeParser.soup.find_all(['sms', 'mms']) else: nodes = mergeParser.soup.find_all('call') for n in nodes: collection.append(n.extract()) return mainParser
def api_parse_sentence(): grammar_name = request.args.get('grammar') tokens = nlp(request.args.get('sentence')) reply = dict(tokens=tokens, grammar=grammar_name) try: try: grammar = grammars[grammar_name] except: raise Exception('Grammar {} not available'.format(grammar_name)) p = parser.Parser(grammar, 'sentences') parses = p.parse(tokens) reply['parses'] = unique(parses, key=lambda parse: parse['data']) if len(reply['parses']) > 20: reply[ 'warning'] = 'There were {} parses, but cut off at {}'.format( len(reply['parses']), 20) reply['parses'] = reply['parses'][:20] return jsonify(reply) except Exception as error: traceback.print_exc() reply['error'] = "{}: {!s}\n{}".format(error.__class__.__name__, error, traceback.format_exc()) response = jsonify(reply) response.status_code = 400 return response
def __init__(self, dspec_path, net_spec, params, auto_mask=True): """ Initialize DataProvider. Args: dspec_path: Path to the dataset specification file. net_spec: Net specification. params: Various options. auto_mask: Whether to automatically generate mask from corresponding label. """ # Params. drange = params['drange'] # Required. dprior = params.get('dprior', None) # Optional. # Build Datasets. print '\n[VolumeDataProvider]' p = parser.Parser(dspec_path, net_spec, params, auto_mask=auto_mask) self.datasets = list() for d in drange: print 'constructing dataset %d...' % d config, dparams = p.parse_dataset(d) dataset = VolumeDataset(config, **dparams) self.datasets.append(dataset) # Sampling weight. self.set_sampling_weights(dprior) # Setup data augmentation. aug_spec = params.get('augment', []) # Default is an empty list. self._data_aug = DataAugmentor(aug_spec)
def test_mnmonic(self): self.parser = parser.Parser(open('test.asm', 'r')) self.parser.advance() self.parser.commandType() self.assertEqual(self.parser.dest(), "D") self.assertEqual(self.parser.comp(), "M") self.assertEqual(self.parser.jump(), "") self.parser.advance() self.parser.commandType() self.assertEqual(self.parser.dest(), "A") self.assertEqual(self.parser.comp(), "M+1") self.assertEqual(self.parser.jump(), "") self.parser.advance() self.parser.commandType() self.assertEqual(self.parser.symbol(), "hoge") self.parser.advance() self.parser.commandType() self.assertEqual(self.parser.symbol(), "LOOP") self.parser.advance() self.parser.commandType() self.assertEqual(self.parser.dest(), "") self.assertEqual(self.parser.comp(), "0") self.assertEqual(self.parser.jump(), "JGT")
def __init__(self, compiler='aspCompiler', silent=False, decoupled=False, agent=False): self.silent = silent self.debug = False self.ignore_errors = False self.ignore_undefined = False self.there_were_roles = False self.roles = [] self.lex = lexer.Lexer() self.lex.build() self.par = parser.Parser() self.par.build() if compiler == 'aspCompiler': self.comp = aspCompiler.AspCompiler(decoupled=decoupled, agent=agent) elif compiler == 'aspReducedCompiler': self.comp = aspReducedCompiler.AspReducedCompiler( decoupled=decoupled, agent=agent) else: if not self.silent: print >> sys.stderr, "Could not find Compiler ", compiler, ". Sticking to aspCompiler." self.comp = aspCompiler.AspCompiler(decoupled=decoupled, agent=agent) self.default_output_path = "./" #"../temp/"
def parse_imports(self, parent_unit: ast.TranslationUnit): for imp_stmt in parent_unit.import_statements: if imp_stmt.path not in self.modules: # Parse # TODO this should be encapsulated more nicely. Currently, same code # as in main.py try: with open(imp_stmt.path) as f: code = f.read() except Exception as e: raise TypecheckException( f"Importing {imp_stmt.path} impossible:" f" '{e}'", imp_stmt) l = lexer.Lexer(code, imp_stmt.path) p = parser.Parser(l) child_unit = p.compile() # add2modmap if not isinstance(child_unit, ast.TranslationUnit): raise TypecheckException( f"Importing {imp_stmt.path}" " failed.", imp_stmt) self.modules[imp_stmt.path] = child_unit # Recurse self.parse_imports(child_unit) # Add to symbol table parent_unit.symbols_global[imp_stmt.name] = bongtypes.Module( imp_stmt.path)
def test_num_lines(self): """Send some 'canned' output to the server and parse the result. Really a test for the parser, but easier to implement here. Just verifying the parser can handle some "real" nethack output, even if it's unpredictable.""" parser_ = parser.Parser() output = list() # for debugging def parse_loop(): """Read and parse until the read timeout""" while True: data = self.server.read() if not data: break #print(data) output.append(data) # debugging parser_.parse_bytes(data) self.assertTrue(parser_.end_of_data) parse_loop() self.server.write(b'y') # shall I pick your class parse_loop() self.server.write(b'y') # is this OK? parse_loop() self.server.write(b' ') # Go bravely! parse_loop() self.server.write(b' ') # Welcome to nethack parse_loop() self.server.write(b'?') # Help command parse_loop() self.server.write(b'i') # list of all commands parse_loop()
def run(self, vmFilename): p = parser.Parser(vmFilename) self.cw.setStaticPrefix(vmFilename) p.advance() while p.hasMoreCommands(): commandType = p.commandType() if commandType == command_types.C_PUSH or commandType == command_types.C_POP: self.cw.WritePushPop(commandType, p.arg1(), p.arg2()) elif commandType == command_types.C_ARITHMETIC: self.cw.WriteArithmetic(p.arg1()) elif commandType == command_types.C_LABEL: self.cw.WriteLabel(p.arg1()) elif commandType == command_types.C_GOTO: self.cw.WriteGoto(p.arg1()) elif commandType == command_types.C_IF: self.cw.WriteIf(p.arg1()) elif commandType == command_types.C_FUNCTION: self.cw.WriteFunction(p.arg1(), p.arg2()) elif commandType == command_types.C_RETURN: self.cw.WriteReturn() elif commandType == command_types.C_CALL: self.cw.WriteCall(p.arg1(), p.arg2()) else: # Unimplemented pass p.advance()
def vmtranslate(self, file, code): """ called from main function to translate vm code """ parsed_lines = parser.Parser(code) while parsed_lines.hasMoreCommands(): parsed_lines.advance() command_type = parsed_lines.getCommandType() if command_type == 'C_ARITHMETIC': self.cwriter.writeArithmetic(parsed_lines.getCommand()) elif command_type == 'C_PUSH' or command_type == 'C_POP': segment = parsed_lines.getArg1() value = parsed_lines.getArg2() self.cwriter.writePushPop(command_type,segment,value) elif command_type == "C_GOTO": self.cwriter.writeGoto(parsed_lines.getArg1()) elif command_type == "C_IF": self.cwriter.writeIf(parsed_lines.getArg1()) elif command_type == "C_LABEL": self.cwriter.writeLabel(parsed_lines.getArg1()) elif command_type == "C_RETURN": self.cwriter.writeReturn() elif command_type == "C_FUNCTION": self.cwriter.writeFunction(parsed_lines.getArg1(), parsed_lines.getArg2()) elif command_type == "C_CALL": self.cwriter.writeCall(parsed_lines.getArg1(), parsed_lines.getArg2())
def TestReturnStatements(self): print("Running TestReturnStatements ...") @dataclass class test: input: str expectedValue: object tests = [ test("return 5", 5), test("return true", True), test("return y", "y"), ] for tt in tests: l = lxr.Lexer(tt.input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if program is None: raise Exception('ParseProgram() returned None') if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}') stmt = program.Statements[0] if not self.testReturnStatement(stmt): return val = stmt.Value if not self.testLiteralExpression(val, tt.expectedValue): return
def TestCallExpressionParsing(self): print("Running TestCallExpressionParsing ...") input = """ add(1, 2 * 3, 4 + 5) """ l = lxr.Lexer(input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statement got={len(program.Statements)}') stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') funcCall = stmt.Expression if not isinstance(funcCall, ast.CallExpression): raise Exception(f'stmt.Expression is not a ast.CallExpression got={stmt.Expression}') if not self.testIdentifier(funcCall.Function, "add"): return if len(function.Arguments) != 3: raise Exception(f'function.Arguments does not contain 3 statements got={stmt.Expression}') self.testLiteralExpression(function.Arguments[0],1) self.testInfixExpression(function.Arguments[1], 2, "*", 3) self.testInfixExpression(function.Arguments[2], 4, "+", 5)
def TestFunctionParameterParsing(self): @dataclass class test: input: str expectedParams: List[str] tests = [ test("fn() {}",[]), test("fn(x) {}",['x']), test("fn(x, y, z) {}",['x', 'y', 'z']) ] for tt in tests: l = lxr.Lexer(tt.input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') function = stmt.Expression if len(function.Parameters) != len(tt.expectedParams): raise Exception(f'function.Parameters does not contain {len(tt.expectedParams)} parameters got {len(function.Parameters)}') ix = 0 for p in tt.expectedParams: self.testLiteralExpression(function.Parameters[ix], p) ix += 1
def TestIntegerLiteralExpressions(self): print("Running TestIntegerLiteralExpressions ...") input = """ 5 """ l = lxr.Lexer(input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}') stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') literal = stmt.Expression if not isinstance(literal, ast.IntegerLiteral): raise Exception(f'Expression is not a ast.Identifier got={stmt.Expression}') if literal.Value != 5: raise Exception(f'ident.Value is not 5 got={literal.Value}') if literal.TokenLiteral() != "5": raise Exception(f'ident.TokenLiteral() is not "5" got={literal.TokenLiteral()}')
def TestBooleanExpressions(self): print("Running TestBooleanExpressions ...") input = """ true """ l = lxr.Lexer(input) p = prsr.Parser(l) program = p.ParseProgram() self.checkParsersErrors(p) if len(program.Statements) != 1: raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}') stmt = program.Statements[0] if not isinstance(stmt, ast.ExpressionStatement): raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}') ident = stmt.Expression if not isinstance(ident, ast.Boolean): raise Exception(f'Expression is not a ast.Boolean got={stmt.Expression}') if ident.Value != True: raise Exception(f'ident.Value is not True got={ident.Value}') if ident.TokenLiteral() != "true": raise Exception(f'ident.TokenLiteral() is not True got={ident.TokenLiteral()}')
def TestParsingInfixExpression(): class struct: def __init__(self, input, leftValue, operator, rightValue): self.input = input self.leftValue = leftValue self.operator = operator self.rightValue = rightValue infixTests = [struct("5 + 5;", 5, "+", 5), struct("5 - 5;", 5, "-", 5), struct("5 * 5;", 5, "*", 5), struct("5 / 5;", 5, "/", 5), struct("5 > 5;", 5, ">", 5), struct("5 < 5;", 5, "<", 5), struct("5 != 5;", 5, "!=", 5), struct("true == true", True, "==", True), struct("true != false", True, "!=", False)] for test in infixTests: l = lexer.Lexer(test.input) p = parser.Parser(l) program = p.ParseProgram() checkParserErrors(p) assert len(program.Statements) == 1 stmt = program.Statements[0] assert isinstance(stmt, ast.ExpressionStatement) exp = stmt.Expression TestIntegerLiteral(exp.Left, test.leftValue) assert exp.Operator == test.operator TestIntegerLiteral(exp.Right, test.rightValue) print("TestParsingInfixExpression test is success")
def test_parser(self): """Send some 'canned' output to the server and parse the result. Really a test for the parser, but easier to implement here. Just verifying the parser can handle some "real" nethack output, even if it's unpredictable.""" parser_ = parser.Parser() output = list() # for debugging def parse_loop(): """Read and parse until the read timeout""" while True: data = self.server.read() if not data: break #print(data) output.append(data) # debugging parser_.parse_bytes(data) self.assertTrue(parser_.end_of_data) parse_loop() self.server.write(b'y') parse_loop() self.server.write(b'y') parse_loop() self.server.write(b'\n') parse_loop()
def run(self, filename): print("Filename: " + str(filename)) # Second pass outputFilename = os.path.splitext(filename)[0] + ".asm" print("outputFilename: " + str(outputFilename)) p = parser.Parser(filename) cw = code_writer.CodeWriter(outputFilename) print("Start advance") p.advance() while p.hasMoreCommands(): commandType = p.commandType() if commandType == command_types.C_PUSH or commandType == command_types.C_POP: cw.WritePushPop(commandType, p.arg1(), p.arg2()) elif commandType == command_types.C_ARITHMETIC: cw.WriteArithmetic(p.arg1()) else: # Unimplemented pass p.advance() print("Finished") cw.Close()
def helper_function(fName, **kwargs): dimension = kwargs['dimension'] toOpen_indep = open(fName) if os.path.exists((fName.replace('.out', '.tdm.out'))): toOpen_dep = fName.replace('.out', '.tdm.out') else: toOpen_dep = '' if os.path.exists((fName.replace('.out', '.geometry'))): toOpen_geometry = fName.replace('.out', '.geometry') else: toOpen_geometry = '' p = parser.Parser(toOpen_indep.read(), dimension=dimension, tdm=toOpen_dep, geometry=toOpen_geometry) try: p.get_rho() except: print 'bad file = ', fName os.remove(fName) toOpen_indep.close() return p
def __init__(self): super(Translator,self).__init__() # Parser module to generate the AST, the symbol table and other data structs. self.Parser = parser.Parser() # Coords for the last AST node visited self.lastInputCoords = '' # coords, example: ':245' self.currentInputLineNumber = 0 # line numbers extracted from coords # Coords last read from a linemarker self.lastinputfile = '' # input file in the last linemarker self.lastinputlineno = 0 # line number since last linemarker self.lastoutputlineno = 0 # line number in output # Stacks of ongoing recursive visits self.stack = [] # class names, example: ['FileAST', 'FuncDef', 'Compound', 'Compound', 'If', 'BinaryOp', 'ArrayRef', 'ID'] self.stacknodes = [] # AST nodes self.currentFunct = '' # name of the function being parsed ('' = none) self.lines = [] # Input<->Output linemaps self.inputtooutput = {} self.outputtoinput = {}
def compilePage(self, filename, docrootPath, defaultOutputEncoding=None, defaultPageTemplate=None): self.declarations = {} self.imports = [] self.gobbleWS = True self.outputEncoding_fromChild = False self.contentType = None self.contentDisposition = None self.allowCaching = None self.indentChars = '\t' # we use tabs for indentation self.session = None self.authorized = None # authorized roles self.authmethod = None # authentication method self.authmethodargs = None # args for the auth method self.errorpage = None # custom error page self.pagemethod = "create" # name used for the page creation method self.baseclasses = [] self.methods = [] self.pageTemplate = defaultPageTemplate self.pageTemplateArgs = {} self.docrootPath = docrootPath self.inputEncoding, self.outputEncoding = parser.determineInputOutputEncodings( filename) if not self.outputEncoding: self.outputEncoding = defaultOutputEncoding self.tpl_outputEncoding = self.tpl_contentType = None try: try: # 1. open the page source file pageSourceStream = None if self.inputEncoding: import codecs pageSourceStream = codecs.open(filename, mode="rb", encoding=self.inputEncoding) else: # use regular file because input encoding is default (not specified). pageSourceStream = file(filename, "r") # 2. parse the page source parse = parser.Parser(pageSourceStream, filename, self.inputEncoding) syntaxtree = parse.parse() self.outputEncoding_fromChild = parse.outputEncoding_fromChild finally: # 3. close the page source file if pageSourceStream: pageSourceStream.close( ) # otherwise .y file remains opened until GC'd. del pageSourceStream # 4. generate python code (result, indent) = self.generateCode(syntaxtree, 0) return result # return the Unicode string except (TokenizerError, parser.ParserError), px: sys.exc_clear() raise CompilerError(px)
def documentation_message(self, package, fun, channel_id, thread_id): """ Create and send a message containing a short description of the function and a link to the RDocumentation website Parameters ---------- package : str name of the R package fun : str name of the R function in the package channel_id : str id of the Slack channel where to post the message thread_id : str id of the Slack thread where to post the message """ desc = parser.Parser(package, fun) desc.retrieve_desc() msg = message.Message() fun_str = package + '::' + fun + '()' msg.create_attachments(desc.text, desc.url, fun_str) post_message = self.client.api_call("chat.postMessage", channel=channel_id, thread_ts=thread_id, icon_emoji=self.emoji, attachments=msg.attachments )
def parseSearchString(searchString): p = parser.Parser(searchString) a = p.parse([{ "token": "(", "types": ["Open Parenthesses", "Block", "Parenthesses"] }, { "token": ")", "types": ["Close Parenthesses", "Block", "Parenthesses"] }, { "token": " ", "types": ["Whitespace"] }, { "token": '"', "types": ["Double quote", "Block"] }, { "token": "+", "types": ["Plus", "Plusminus"] }, { "token": "-", "types": ["Minus", "Plusminus"] }, { "token": ",", "types": ["Comma"] }], []) # , parser.tokenPlusMinus, parser.tokenComma, # parser.tokenWhiteSpace, parser.tokenDefault], []) # [parser.synQuotes, parser.synWhitespaces, # parser.synEmptyBlocks, # parser.synPlusMinus]) p.str() return (a)
def parse_pairwise_function(function_name, column=True, M_c=None, column_lists={}): ##TODO move to parser if column: if function_name == 'mutual information': return functions._mutual_information elif function_name == 'dependence probability': return functions._dependence_probability elif function_name == 'correlation': return functions._correlation else: raise utils.BayesDBParseError('Invalid column function: %s' % function_name) else: # TODO: need to refactor to support similarity with respect to column, because then we need to parse # and return the column id here. ##TODO temporary hack - move to parser maybe combine with parse_functions p = parser.Parser() _, target_columns = p.get_args_similarity(function_name, M_c, None, M_c, None, column_lists) if target_columns is None: return (functions._similarity, None) elif type(target_columns) == list: return (functions._similarity, target_columns) else: raise utils.BayesDBParseError('Invalid row function: %s' % function_name)
def main(): sv_tags = [ "<ROOT>", "ADJ", "ADP", "ADV", "AUX", "CONJ", "DET", "INTJ", "NOUN", "NUM", "PART", "PRON", "PROPN", "PUNCT", "SCONJ", "SYM", "VERB" ] en_tags = [ "<ROOT>", "ADJ", "ADP", "ADV", "AUX", "CONJ", "DET", "INTJ", "NOUN", "NUM", "PART", "PRON", "PROPN", "PUNCT", "SCONJ", "SYM", "VERB", "X" ] arc_tags = [ '<ROOT>', 'name', 'nsubjpass', 'dobj', 'acl', 'advcl', 'advmod', 'amod', 'appos', 'aux', 'case', 'cc', 'ccomp', 'clf', 'compound', 'conj', 'cop', 'csubj', 'dep', 'det', 'discourse', 'det', 'discourse', 'dislocated', 'expl', 'fixed', 'flat', 'goeswith', 'iobj', 'list', 'mark', 'nmod', 'nsubj', 'nummod', 'obj', 'obl', 'orphan', 'parataxis', 'punct', 'reparandum', 'root', 'vocative', 'xcomp' ] sv_train_file = "treebanks/sv_train.conllu" sv_test_file = "treebanks/sv_dev.conllu" en_train_file = "treebanks/en_train.conllu" en_test_file = "treebanks/en_dev.conllu" #model = wsm.WSM() #model.create_model(en_train_file) model_path = 'models/wsm_en' myTagger = tagger.Tagger(en_tags) arc_tagger = tagger.Tagger(arc_tags) myParser = parser.Parser(myTagger, arc_tagger, model_path) dataReader.evaluate(en_train_file, en_test_file, myParser) dataReader.evaluate(sv_train_file, sv_test_file, myParser)
def eval_test(input: str) -> MonkeyObject: l = lexer.Lexer(input) p = parser.Parser(l) program = p.parse_program() env = mobject.Environment() return evaluator.eval(program, env)
def run(self, source): scanner = scn.Scanner(self, source) tokens = scanner.scan_tokens() parser = prs.Parser(self, tokens) expression = parser.parse() self.interpreter.interpret(expression)
def __init__(self, filepath): """Creates a new instance. @param filepath: path to PCAP file """ self.filepath = filepath # List of all hosts. self.hosts = [] # List containing all non-private IP addresses. self.unique_hosts = [] # List of unique domains. self.unique_domains = [] # List containing all TCP packets. self.tcp_connections = [] self.tcp_connections_seen = set() # List containing all UDP packets. self.udp_connections = [] self.udp_connections_seen = set() # List containing all ICMP requests. self.icmp_requests = [] # List containing all HTTP requests. self.http_requests = {} # List containing all DNS requests. self.dns_requests = {} self.dns_answers = set() # List containing all SMTP requests. self.smtp_requests = [] # Reconstruncted SMTP flow. self.smtp_flow = {} # List containing all IRC requests. self.irc_requests = [] # Dictionary containing all the results of this processing. self.results = {} # Parser object that parse PCAP filo to Json report self.parser = parser.Parser(filepath)
def __init__(self): # 实例化 管理器、下载器、解析器、输出器 self.manager = manager.Manager() self.download = download.Download() self.parser = parser.Parser() self.output = output.Output()
def test_parsing_infix_expressions(): infix_tests = [ ("5 + 5", 5, "+", 5), ("5 - 5", 5, "-", 5), ("5*5", 5, "*", 5), ("5 / 5", 5, "/", 5), ("5 >5", 5, ">", 5), ("5< 5", 5, "<", 5), ("5 == 5", 5, "==", 5), ("5!=5", 5, "!=", 5), ("true == true", True, "==", True), ("true!=false", True, "!=", False), ] for input, left_value, operator, right_value in infix_tests: l = lexer.Lexer(input) p = parser.Parser(l) program = p.parse_program() check_parser_errors(p) check_program_length(program, 1) statement = program.statements[0] check_statement(statement, ast.ExpressionStatement) exp = statement.expression infix_expression_test(exp, left_value, operator, right_value)