예제 #1
0
def diff(source1, source2, dest, config):
    IDS = config["IDS"]
    REQUIRED_ATTRS = config["REQUIRED_ATTRS"]
    (fname1, need_cleanup1) = process_source(source1)
    (fname2, need_cleanup2) = process_source(source2)
    try:
        p = parser.Parser()
        parsed1 = p.parse_file(open(fname1))
        p = parser.Parser()
        parsed2 = p.parse_file(open(fname2))
        dest = open(dest, "w")
        diffed = parsed1.diff(parsed2,
                              path="",
                              ids=IDS,
                              required=REQUIRED_ATTRS)
        del parsed1, parsed2

        writer = xmlwriter.XMLWriter(dest, encoding="utf-8")
        writer.start_document()
        saxxml.diff_tree2xml(writer, diffed, required_attrs=REQUIRED_ATTRS)
        writer.end_document()

    except Exception:
        raise
    finally:
        if need_cleanup1:
            os.remove(fname1)
        if need_cleanup2:
            os.remove(fname2)
예제 #2
0
def loadFiles(fnameList):
    mainParser = None

    with open(fnameList[0], 'rb') as f:
        mainParser = parser.Parser(f.read())
        if mainParser.smsXML:
            collection = mainParser.soup.find('smses')
        else:
            collection = mainParser.soup.find('calls')

    if len(fnameList) > 1:
        for i in range(1, len(fnameList)):
            with open(fnameList[i], 'rb') as f:
                mergeParser = parser.Parser(f.read())
                if mergeParser.smsXML != mainParser.smsXML:
                    raise Exception('cannot merge mixed file types')

                if mainParser.smsXML:
                    nodes = mergeParser.soup.find_all(['sms', 'mms'])
                else:
                    nodes = mergeParser.soup.find_all('call')

                for n in nodes:
                    collection.append(n.extract())

    return mainParser
예제 #3
0
def api_parse_sentence():
    grammar_name = request.args.get('grammar')
    tokens = nlp(request.args.get('sentence'))
    reply = dict(tokens=tokens, grammar=grammar_name)

    try:
        try:
            grammar = grammars[grammar_name]
        except:
            raise Exception('Grammar {} not available'.format(grammar_name))

        p = parser.Parser(grammar, 'sentences')
        parses = p.parse(tokens)
        reply['parses'] = unique(parses, key=lambda parse: parse['data'])

        if len(reply['parses']) > 20:
            reply[
                'warning'] = 'There were {} parses, but cut off at {}'.format(
                    len(reply['parses']), 20)
            reply['parses'] = reply['parses'][:20]

        return jsonify(reply)
    except Exception as error:
        traceback.print_exc()
        reply['error'] = "{}: {!s}\n{}".format(error.__class__.__name__, error,
                                               traceback.format_exc())
        response = jsonify(reply)
        response.status_code = 400
        return response
예제 #4
0
    def __init__(self, dspec_path, net_spec, params, auto_mask=True):
        """
        Initialize DataProvider.

        Args:
            dspec_path: Path to the dataset specification file.
            net_spec:   Net specification.
            params:     Various options.
            auto_mask:  Whether to automatically generate mask from
                        corresponding label.
        """
        # Params.
        drange = params['drange']  # Required.
        dprior = params.get('dprior', None)  # Optional.

        # Build Datasets.
        print '\n[VolumeDataProvider]'
        p = parser.Parser(dspec_path, net_spec, params, auto_mask=auto_mask)
        self.datasets = list()
        for d in drange:
            print 'constructing dataset %d...' % d
            config, dparams = p.parse_dataset(d)
            dataset = VolumeDataset(config, **dparams)
            self.datasets.append(dataset)

        # Sampling weight.
        self.set_sampling_weights(dprior)

        # Setup data augmentation.
        aug_spec = params.get('augment', [])  # Default is an empty list.
        self._data_aug = DataAugmentor(aug_spec)
예제 #5
0
    def test_mnmonic(self):
        self.parser = parser.Parser(open('test.asm', 'r'))

        self.parser.advance()
        self.parser.commandType()
        self.assertEqual(self.parser.dest(), "D")
        self.assertEqual(self.parser.comp(), "M")
        self.assertEqual(self.parser.jump(), "")

        self.parser.advance()
        self.parser.commandType()
        self.assertEqual(self.parser.dest(), "A")
        self.assertEqual(self.parser.comp(), "M+1")
        self.assertEqual(self.parser.jump(), "")

        self.parser.advance()
        self.parser.commandType()
        self.assertEqual(self.parser.symbol(), "hoge")

        self.parser.advance()
        self.parser.commandType()
        self.assertEqual(self.parser.symbol(), "LOOP")

        self.parser.advance()
        self.parser.commandType()
        self.assertEqual(self.parser.dest(), "")
        self.assertEqual(self.parser.comp(), "0")
        self.assertEqual(self.parser.jump(), "JGT")
예제 #6
0
    def __init__(self,
                 compiler='aspCompiler',
                 silent=False,
                 decoupled=False,
                 agent=False):

        self.silent = silent
        self.debug = False
        self.ignore_errors = False
        self.ignore_undefined = False
        self.there_were_roles = False
        self.roles = []

        self.lex = lexer.Lexer()
        self.lex.build()
        self.par = parser.Parser()
        self.par.build()

        if compiler == 'aspCompiler':
            self.comp = aspCompiler.AspCompiler(decoupled=decoupled,
                                                agent=agent)
        elif compiler == 'aspReducedCompiler':
            self.comp = aspReducedCompiler.AspReducedCompiler(
                decoupled=decoupled, agent=agent)
        else:
            if not self.silent:
                print >> sys.stderr, "Could not find Compiler ", compiler, ". Sticking to aspCompiler."
            self.comp = aspCompiler.AspCompiler(decoupled=decoupled,
                                                agent=agent)

        self.default_output_path = "./"  #"../temp/"
예제 #7
0
 def parse_imports(self, parent_unit: ast.TranslationUnit):
     for imp_stmt in parent_unit.import_statements:
         if imp_stmt.path not in self.modules:
             # Parse
             # TODO this should be encapsulated more nicely. Currently, same code
             # as in main.py
             try:
                 with open(imp_stmt.path) as f:
                     code = f.read()
             except Exception as e:
                 raise TypecheckException(
                     f"Importing {imp_stmt.path} impossible:"
                     f" '{e}'", imp_stmt)
             l = lexer.Lexer(code, imp_stmt.path)
             p = parser.Parser(l)
             child_unit = p.compile()
             # add2modmap
             if not isinstance(child_unit, ast.TranslationUnit):
                 raise TypecheckException(
                     f"Importing {imp_stmt.path}"
                     " failed.", imp_stmt)
             self.modules[imp_stmt.path] = child_unit
             # Recurse
             self.parse_imports(child_unit)
         # Add to symbol table
         parent_unit.symbols_global[imp_stmt.name] = bongtypes.Module(
             imp_stmt.path)
예제 #8
0
    def test_num_lines(self):
        """Send some 'canned' output to the server and parse the result. Really
           a test for the parser, but easier to implement here. Just verifying
           the parser can handle some "real" nethack output, even if it's
           unpredictable."""
        parser_ = parser.Parser()
        output = list()  # for debugging

        def parse_loop():
            """Read and parse until the read timeout"""
            while True:
                data = self.server.read()
                if not data:
                    break
                #print(data)
                output.append(data)  # debugging
                parser_.parse_bytes(data)
            self.assertTrue(parser_.end_of_data)

        parse_loop()
        self.server.write(b'y')  # shall I pick your class
        parse_loop()
        self.server.write(b'y')  # is this OK?
        parse_loop()
        self.server.write(b' ')  # Go bravely!
        parse_loop()
        self.server.write(b' ')  # Welcome to nethack
        parse_loop()
        self.server.write(b'?')  # Help command
        parse_loop()
        self.server.write(b'i')  # list of all commands
        parse_loop()
예제 #9
0
    def run(self, vmFilename):

        p = parser.Parser(vmFilename)
        self.cw.setStaticPrefix(vmFilename)

        p.advance()
        while p.hasMoreCommands():

            commandType = p.commandType()

            if commandType == command_types.C_PUSH or commandType == command_types.C_POP:
                self.cw.WritePushPop(commandType, p.arg1(), p.arg2())
            elif commandType == command_types.C_ARITHMETIC:
                self.cw.WriteArithmetic(p.arg1())
            elif commandType == command_types.C_LABEL:
                self.cw.WriteLabel(p.arg1())
            elif commandType == command_types.C_GOTO:
                self.cw.WriteGoto(p.arg1())
            elif commandType == command_types.C_IF:
                self.cw.WriteIf(p.arg1())
            elif commandType == command_types.C_FUNCTION:
                self.cw.WriteFunction(p.arg1(), p.arg2())
            elif commandType == command_types.C_RETURN:
                self.cw.WriteReturn()
            elif commandType == command_types.C_CALL:
                self.cw.WriteCall(p.arg1(), p.arg2())
            else:
                # Unimplemented
                pass

            p.advance()
예제 #10
0
	def vmtranslate(self, file, code):
		"""
		called from main function to translate vm code
		"""

		parsed_lines = parser.Parser(code)

		while parsed_lines.hasMoreCommands():
			
			parsed_lines.advance()
			command_type = parsed_lines.getCommandType()
	

			if command_type == 'C_ARITHMETIC':
				self.cwriter.writeArithmetic(parsed_lines.getCommand())
			elif command_type == 'C_PUSH' or command_type == 'C_POP':
				segment = parsed_lines.getArg1()
				value = parsed_lines.getArg2()
				self.cwriter.writePushPop(command_type,segment,value)
			elif command_type == "C_GOTO":
				self.cwriter.writeGoto(parsed_lines.getArg1())
			elif command_type == "C_IF":
				self.cwriter.writeIf(parsed_lines.getArg1())
			elif command_type == "C_LABEL":
				self.cwriter.writeLabel(parsed_lines.getArg1())
			elif command_type == "C_RETURN":
				self.cwriter.writeReturn()
			elif command_type == "C_FUNCTION":
				self.cwriter.writeFunction(parsed_lines.getArg1(), parsed_lines.getArg2())
			elif command_type == "C_CALL":
				self.cwriter.writeCall(parsed_lines.getArg1(), parsed_lines.getArg2())
예제 #11
0
    def TestReturnStatements(self):
        print("Running TestReturnStatements ...")        

        @dataclass
        class test:
            input: str
            expectedValue: object

        tests = [
            test("return 5", 5),
            test("return true", True),
            test("return y", "y"),

        ]  

        for tt in tests:         
            l = lxr.Lexer(tt.input)
            p = prsr.Parser(l)

            program = p.ParseProgram()
            self.checkParsersErrors(p)

            if program is None:
                raise Exception('ParseProgram() returned None')
        
            if len(program.Statements) != 1:
                raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}')
        
            stmt = program.Statements[0]
            if not self.testReturnStatement(stmt):
                return 

            val = stmt.Value
            if not self.testLiteralExpression(val, tt.expectedValue):
                return
예제 #12
0
    def TestCallExpressionParsing(self):
        print("Running TestCallExpressionParsing ...")
        input = """
        add(1, 2 * 3, 4 + 5)
        """     
        
        l = lxr.Lexer(input)
        p = prsr.Parser(l)

        program = p.ParseProgram()
        self.checkParsersErrors(p)

        if len(program.Statements) != 1:
            raise Exception(f'program.Statements does not contain 1 statement got={len(program.Statements)}')

        stmt = program.Statements[0]
        if not isinstance(stmt, ast.ExpressionStatement):
            raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}')        

        funcCall = stmt.Expression
        if not isinstance(funcCall, ast.CallExpression):
            raise Exception(f'stmt.Expression is not a ast.CallExpression got={stmt.Expression}') 

        if not self.testIdentifier(funcCall.Function, "add"):
            return 

        if len(function.Arguments) != 3:
            raise Exception(f'function.Arguments does not contain 3 statements got={stmt.Expression}') 

        self.testLiteralExpression(function.Arguments[0],1)
        self.testInfixExpression(function.Arguments[1], 2, "*", 3)
        self.testInfixExpression(function.Arguments[2], 4, "+", 5)
예제 #13
0
    def TestFunctionParameterParsing(self):
        @dataclass
        class test:
            input: str
            expectedParams: List[str]
        
        tests = [
            test("fn() {}",[]),
            test("fn(x) {}",['x']),
            test("fn(x, y, z) {}",['x', 'y', 'z'])
        ]

        for tt in tests:
            l = lxr.Lexer(tt.input)
            p = prsr.Parser(l)

            program = p.ParseProgram()
            self.checkParsersErrors(p)     

            stmt = program.Statements[0]       

            if not isinstance(stmt, ast.ExpressionStatement):
                raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}')
            
            function = stmt.Expression

            if len(function.Parameters) != len(tt.expectedParams):
                raise Exception(f'function.Parameters does not contain {len(tt.expectedParams)} parameters got {len(function.Parameters)}')

            ix = 0
            for p in tt.expectedParams:
                self.testLiteralExpression(function.Parameters[ix], p)
                ix += 1
예제 #14
0
    def TestIntegerLiteralExpressions(self):
        print("Running TestIntegerLiteralExpressions ...")
        input = """
        5
        """

        l = lxr.Lexer(input)
        p = prsr.Parser(l)

        program = p.ParseProgram()
        self.checkParsersErrors(p)

        if len(program.Statements) != 1:
            raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}')

        stmt = program.Statements[0]
        if not isinstance(stmt, ast.ExpressionStatement):
            raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}')        
                
        
        literal = stmt.Expression
        if not isinstance(literal, ast.IntegerLiteral):
            raise Exception(f'Expression is not a ast.Identifier got={stmt.Expression}')        

        if literal.Value != 5:
            raise Exception(f'ident.Value is not 5 got={literal.Value}')

        if literal.TokenLiteral() != "5":
            raise Exception(f'ident.TokenLiteral() is not "5" got={literal.TokenLiteral()}')                    
예제 #15
0
    def TestBooleanExpressions(self):
        print("Running TestBooleanExpressions ...")
        input = """
        true
        """     

        l = lxr.Lexer(input)
        p = prsr.Parser(l)

        program = p.ParseProgram()
        self.checkParsersErrors(p)

        if len(program.Statements) != 1:
            raise Exception(f'program.Statements does not contain 1 statements. got={len(program.Statements)}')

        stmt = program.Statements[0]
        if not isinstance(stmt, ast.ExpressionStatement):
            raise Exception(f'statement is not a ast.ExpressionStatement got={program.Statements[0]}')        
                
        
        ident = stmt.Expression
        if not isinstance(ident, ast.Boolean):
            raise Exception(f'Expression is not a ast.Boolean got={stmt.Expression}')        

        if ident.Value != True:
            raise Exception(f'ident.Value is not True got={ident.Value}')

        if ident.TokenLiteral() != "true":
            raise Exception(f'ident.TokenLiteral() is not True got={ident.TokenLiteral()}')           
예제 #16
0
def TestParsingInfixExpression():
    class struct:
        def __init__(self, input, leftValue, operator, rightValue):
            self.input = input
            self.leftValue = leftValue
            self.operator = operator
            self.rightValue = rightValue
    infixTests = [struct("5 + 5;", 5, "+", 5),
                  struct("5 - 5;", 5, "-", 5),
                  struct("5 * 5;", 5, "*", 5),
                  struct("5 / 5;", 5, "/", 5),
                  struct("5 > 5;", 5, ">", 5),
                  struct("5 < 5;", 5, "<", 5),
                  struct("5 != 5;", 5, "!=", 5),
                  struct("true == true", True, "==", True),
                  struct("true != false", True, "!=", False)]
    for test in infixTests:
        l = lexer.Lexer(test.input)
        p = parser.Parser(l)
        program = p.ParseProgram()
        checkParserErrors(p)
        assert len(program.Statements) == 1
        stmt = program.Statements[0]
        assert isinstance(stmt, ast.ExpressionStatement)
        exp = stmt.Expression
        TestIntegerLiteral(exp.Left, test.leftValue)
        assert exp.Operator == test.operator
        TestIntegerLiteral(exp.Right, test.rightValue)
    print("TestParsingInfixExpression test is success")
예제 #17
0
    def test_parser(self):
        """Send some 'canned' output to the server and parse the result. Really
           a test for the parser, but easier to implement here. Just verifying
           the parser can handle some "real" nethack output, even if it's
           unpredictable."""
        parser_ = parser.Parser()
        output = list()  # for debugging

        def parse_loop():
            """Read and parse until the read timeout"""
            while True:
                data = self.server.read()
                if not data:
                    break
                #print(data)
                output.append(data)  # debugging
                parser_.parse_bytes(data)
            self.assertTrue(parser_.end_of_data)

        parse_loop()
        self.server.write(b'y')
        parse_loop()
        self.server.write(b'y')
        parse_loop()
        self.server.write(b'\n')
        parse_loop()
예제 #18
0
    def run(self, filename):

        print("Filename: " + str(filename))

        # Second pass
        outputFilename = os.path.splitext(filename)[0] + ".asm"
        print("outputFilename: " + str(outputFilename))

        p = parser.Parser(filename)
        cw = code_writer.CodeWriter(outputFilename)

        print("Start advance")

        p.advance()
        while p.hasMoreCommands():

            commandType = p.commandType()

            if commandType == command_types.C_PUSH or commandType == command_types.C_POP:
                cw.WritePushPop(commandType, p.arg1(), p.arg2())
            elif commandType == command_types.C_ARITHMETIC:
                cw.WriteArithmetic(p.arg1())
            else:
                # Unimplemented
                pass

            p.advance()

        print("Finished")
        cw.Close()
예제 #19
0
def helper_function(fName, **kwargs):
    dimension = kwargs['dimension']
    toOpen_indep = open(fName)

    if os.path.exists((fName.replace('.out', '.tdm.out'))):
        toOpen_dep = fName.replace('.out', '.tdm.out')
    else:
        toOpen_dep = ''

    if os.path.exists((fName.replace('.out', '.geometry'))):
        toOpen_geometry = fName.replace('.out', '.geometry')
    else:
        toOpen_geometry = ''

    p = parser.Parser(toOpen_indep.read(),
                      dimension=dimension,
                      tdm=toOpen_dep,
                      geometry=toOpen_geometry)
    try:
        p.get_rho()
    except:
        print 'bad file = ', fName
        os.remove(fName)
    toOpen_indep.close()
    return p
예제 #20
0
	def __init__(self):
		super(Translator,self).__init__()

		# Parser module to generate the AST, the symbol table and other data structs.
		self.Parser = parser.Parser()

		# Coords for the last AST node visited
		self.lastInputCoords = ''         # coords, example: ':245'
		self.currentInputLineNumber = 0   # line numbers extracted from coords

		# Coords last read from a linemarker
		self.lastinputfile = ''      # input file in the last linemarker
		self.lastinputlineno = 0     # line number since last linemarker
		self.lastoutputlineno = 0    # line number in output

		# Stacks of ongoing recursive visits
		self.stack = []       # class names, example: ['FileAST', 'FuncDef', 'Compound', 'Compound', 'If', 'BinaryOp', 'ArrayRef', 'ID']
		self.stacknodes = []  # AST nodes

		self.currentFunct = '' # name of the function being parsed ('' = none)

		self.lines = []

		# Input<->Output linemaps
		self.inputtooutput = {}
		self.outputtoinput = {}
예제 #21
0
    def compilePage(self,
                    filename,
                    docrootPath,
                    defaultOutputEncoding=None,
                    defaultPageTemplate=None):
        self.declarations = {}
        self.imports = []
        self.gobbleWS = True
        self.outputEncoding_fromChild = False
        self.contentType = None
        self.contentDisposition = None
        self.allowCaching = None
        self.indentChars = '\t'  # we use tabs for indentation
        self.session = None
        self.authorized = None  # authorized roles
        self.authmethod = None  # authentication method
        self.authmethodargs = None  # args for the auth method
        self.errorpage = None  # custom error page
        self.pagemethod = "create"  # name used for the page creation method
        self.baseclasses = []
        self.methods = []
        self.pageTemplate = defaultPageTemplate
        self.pageTemplateArgs = {}
        self.docrootPath = docrootPath
        self.inputEncoding, self.outputEncoding = parser.determineInputOutputEncodings(
            filename)
        if not self.outputEncoding:
            self.outputEncoding = defaultOutputEncoding
        self.tpl_outputEncoding = self.tpl_contentType = None
        try:
            try:
                # 1. open the page source file
                pageSourceStream = None
                if self.inputEncoding:
                    import codecs
                    pageSourceStream = codecs.open(filename,
                                                   mode="rb",
                                                   encoding=self.inputEncoding)
                else:
                    # use regular file because input encoding is default (not specified).
                    pageSourceStream = file(filename, "r")
                # 2. parse the page source
                parse = parser.Parser(pageSourceStream, filename,
                                      self.inputEncoding)
                syntaxtree = parse.parse()
                self.outputEncoding_fromChild = parse.outputEncoding_fromChild
            finally:
                # 3. close the page source file
                if pageSourceStream:
                    pageSourceStream.close(
                    )  # otherwise .y file remains opened until GC'd.
                del pageSourceStream

            # 4. generate python code
            (result, indent) = self.generateCode(syntaxtree, 0)
            return result  # return the Unicode string

        except (TokenizerError, parser.ParserError), px:
            sys.exc_clear()
            raise CompilerError(px)
예제 #22
0
    def documentation_message(self, package, fun, channel_id, thread_id):
        """
        Create and send a message containing a short description of the function
        and a link to the RDocumentation website

        Parameters
        ----------
        package : str
            name of the R package
        fun : str
            name of the R function in the package
        channel_id : str
            id of the Slack channel where to post the message
        thread_id : str
            id of the Slack thread where to post the message

        """

        desc = parser.Parser(package, fun)
        desc.retrieve_desc()

        msg = message.Message()
        fun_str = package + '::' + fun + '()'
        msg.create_attachments(desc.text, desc.url, fun_str)

        post_message = self.client.api_call("chat.postMessage",
                                            channel=channel_id,
                                            thread_ts=thread_id,
                                            icon_emoji=self.emoji,
                                            attachments=msg.attachments
                                            )
예제 #23
0
def parseSearchString(searchString):
    p = parser.Parser(searchString)
    a = p.parse([{
        "token": "(",
        "types": ["Open Parenthesses", "Block", "Parenthesses"]
    }, {
        "token": ")",
        "types": ["Close Parenthesses", "Block", "Parenthesses"]
    }, {
        "token": " ",
        "types": ["Whitespace"]
    }, {
        "token": '"',
        "types": ["Double quote", "Block"]
    }, {
        "token": "+",
        "types": ["Plus", "Plusminus"]
    }, {
        "token": "-",
        "types": ["Minus", "Plusminus"]
    }, {
        "token": ",",
        "types": ["Comma"]
    }], [])

    # , parser.tokenPlusMinus, parser.tokenComma,
    #                  parser.tokenWhiteSpace, parser.tokenDefault], [])

    # [parser.synQuotes, parser.synWhitespaces,
    #                                                                 parser.synEmptyBlocks,
    #                                                                 parser.synPlusMinus])
    p.str()
    return (a)
예제 #24
0
def parse_pairwise_function(function_name,
                            column=True,
                            M_c=None,
                            column_lists={}):  ##TODO move to parser
    if column:
        if function_name == 'mutual information':
            return functions._mutual_information
        elif function_name == 'dependence probability':
            return functions._dependence_probability
        elif function_name == 'correlation':
            return functions._correlation
        else:
            raise utils.BayesDBParseError('Invalid column function: %s' %
                                          function_name)
    else:
        # TODO: need to refactor to support similarity with respect to column, because then we need to parse
        # and return the column id here.
        ##TODO temporary hack - move to parser maybe combine with parse_functions
        p = parser.Parser()
        _, target_columns = p.get_args_similarity(function_name, M_c, None,
                                                  M_c, None, column_lists)
        if target_columns is None:
            return (functions._similarity, None)
        elif type(target_columns) == list:
            return (functions._similarity, target_columns)
        else:
            raise utils.BayesDBParseError('Invalid row function: %s' %
                                          function_name)
예제 #25
0
def main():
    sv_tags = [
        "<ROOT>", "ADJ", "ADP", "ADV", "AUX", "CONJ", "DET", "INTJ", "NOUN",
        "NUM", "PART", "PRON", "PROPN", "PUNCT", "SCONJ", "SYM", "VERB"
    ]
    en_tags = [
        "<ROOT>", "ADJ", "ADP", "ADV", "AUX", "CONJ", "DET", "INTJ", "NOUN",
        "NUM", "PART", "PRON", "PROPN", "PUNCT", "SCONJ", "SYM", "VERB", "X"
    ]
    arc_tags = [
        '<ROOT>', 'name', 'nsubjpass', 'dobj', 'acl', 'advcl', 'advmod',
        'amod', 'appos', 'aux', 'case', 'cc', 'ccomp', 'clf', 'compound',
        'conj', 'cop', 'csubj', 'dep', 'det', 'discourse', 'det', 'discourse',
        'dislocated', 'expl', 'fixed', 'flat', 'goeswith', 'iobj', 'list',
        'mark', 'nmod', 'nsubj', 'nummod', 'obj', 'obl', 'orphan', 'parataxis',
        'punct', 'reparandum', 'root', 'vocative', 'xcomp'
    ]

    sv_train_file = "treebanks/sv_train.conllu"
    sv_test_file = "treebanks/sv_dev.conllu"
    en_train_file = "treebanks/en_train.conllu"
    en_test_file = "treebanks/en_dev.conllu"

    #model = wsm.WSM()
    #model.create_model(en_train_file)

    model_path = 'models/wsm_en'
    myTagger = tagger.Tagger(en_tags)
    arc_tagger = tagger.Tagger(arc_tags)
    myParser = parser.Parser(myTagger, arc_tagger, model_path)
    dataReader.evaluate(en_train_file, en_test_file, myParser)
    dataReader.evaluate(sv_train_file, sv_test_file, myParser)
예제 #26
0
def eval_test(input: str) -> MonkeyObject:
    l = lexer.Lexer(input)
    p = parser.Parser(l)
    program = p.parse_program()
    env = mobject.Environment()

    return evaluator.eval(program, env)
예제 #27
0
    def run(self, source):
        scanner = scn.Scanner(self, source)
        tokens = scanner.scan_tokens()
        parser = prs.Parser(self, tokens)
        expression = parser.parse()

        self.interpreter.interpret(expression)
예제 #28
0
    def __init__(self, filepath):
        """Creates a new instance.
        @param filepath: path to PCAP file
        """
        self.filepath = filepath

        # List of all hosts.
        self.hosts = []
        # List containing all non-private IP addresses.
        self.unique_hosts = []
        # List of unique domains.
        self.unique_domains = []
        # List containing all TCP packets.
        self.tcp_connections = []
        self.tcp_connections_seen = set()
        # List containing all UDP packets.
        self.udp_connections = []
        self.udp_connections_seen = set()
        # List containing all ICMP requests.
        self.icmp_requests = []
        # List containing all HTTP requests.
        self.http_requests = {}
        # List containing all DNS requests.
        self.dns_requests = {}
        self.dns_answers = set()
        # List containing all SMTP requests.
        self.smtp_requests = []
        # Reconstruncted SMTP flow.
        self.smtp_flow = {}
        # List containing all IRC requests.
        self.irc_requests = []
        # Dictionary containing all the results of this processing.
        self.results = {}
        # Parser object that parse PCAP filo to Json report
        self.parser = parser.Parser(filepath)
예제 #29
0
    def __init__(self):

        # 实例化 管理器、下载器、解析器、输出器
        self.manager = manager.Manager()
        self.download = download.Download()
        self.parser = parser.Parser()
        self.output = output.Output()
예제 #30
0
def test_parsing_infix_expressions():
    infix_tests = [
        ("5 + 5", 5, "+", 5),
        ("5 - 5", 5, "-", 5),
        ("5*5", 5, "*", 5),
        ("5 / 5", 5, "/", 5),
        ("5 >5", 5, ">", 5),
        ("5< 5", 5, "<", 5),
        ("5 == 5", 5, "==", 5),
        ("5!=5", 5, "!=", 5),
        ("true == true", True, "==", True),
        ("true!=false", True, "!=", False),
    ]

    for input, left_value, operator, right_value in infix_tests:
        l = lexer.Lexer(input)
        p = parser.Parser(l)
        program = p.parse_program()
        check_parser_errors(p)
        check_program_length(program, 1)

        statement = program.statements[0]
        check_statement(statement, ast.ExpressionStatement)

        exp = statement.expression

        infix_expression_test(exp, left_value, operator, right_value)