Ejemplo n.º 1
0
 def __convert(self,string,dic):
     if not string:
         return string
     def repl(x):
         return "%" if x[0]=="" else dic[x[0]]
     var=QuotedString("%").setParseAction(repl)
     return (var.transformString(string))
Ejemplo n.º 2
0
def transform_human(text, main_window):
    """Transform user input into something Script can read.

    Main window is needed for tool integration."""
    # these are parseActions for pyparsing.
    def str_literal_to_hex(s, loc, toks):
        for i, t in enumerate(toks):
            toks[i] = ''.join(['0x', t.encode('hex')])
        return toks
    def var_name_to_value(s, loc, toks):
        for i, t in enumerate(toks):
            val = main_window.dock_handler.variables.get_key(t.strip('$'))
            if val:
                toks[i] = val
        return toks
    # ^ parseActions for pyparsing end here.
    str_literal = QuotedString('"')
    str_literal.setParseAction(str_literal_to_hex)
    var_name = pyparsing.Combine(Word('$') + Word(pyparsing.alphas))
    var_name.setParseAction(var_name_to_value)

    s = text
    s = var_name.transformString(s)
    s = str_literal.transformString(s)
    return s
Ejemplo n.º 3
0
 def populate(self):
     tokens=self.tokens
     if len(tokens)==2:
         self.containsExternalCode=True
         self.isJustExternalCode=True
         if tokens[0]=='assert':
             self.addReturns=False
             self.throws=False
         elif tokens[0]=='throws':
             self.throws=True
             self.addReturns=False
         elif tokens[0]=='returns':
             self.addReturns=True
             self.throws=False
         else: raise SemanticException('Invalid Partition Check Item')
         self.code=tokens[1]
     elif len(tokens)==3:
         self.addReturns=tokens[0]=='returns'
         self.throws=not self.addReturns
         tok=self.tokens[2]
         self.comparator=self.tokens[1]
         parse=QuotedString ("<%", "\\", None, False, unquoteResults=True, endQuoteChar="%>")
         self.isJustExternalCode=False
         try:
             self.code=parse.parseString(tok)[0]
             self.containsExternalCode=True
         except ParseException:
             self.containsExternalCode=False
             self.code=tok
     else:
         raise SemanticException("Check is invalid")
Ejemplo n.º 4
0
 def create_type_query_syntax(self):
     create_type_keyword = CaselessLiteral("CREATE TYPE").setParseAction(self.create_new_type_query_obj)
     new_type = Word(alphas).setParseAction(self.set_type)
     parent_type = Word(alphas).setParseAction(self.set_parent_type)
     description = QuotedString("'", multiline=True) ^ QuotedString('"', multiline=True)
     description.setParseAction(self.set_description)
     create_type_query = create_type_keyword + new_type + parent_type + description
     return create_type_query
Ejemplo n.º 5
0
def _define_grammar():
    """
    Creates and returns a copy of the selector grammar.

    Wrapped in a function to avoid polluting the module namespace.
    """
    expr = Forward()

    label_name = Word(LABEL_CHARS)
    label_name.setParseAction(LabelNode)

    string_literal = QuotedString('"') | QuotedString("'")
    string_literal.setParseAction(LiteralNode)

    set_literal = (Suppress("{") +
                   delimitedList(QuotedString('"') | QuotedString("'"), ",") +
                   Suppress("}"))
    set_literal.setParseAction(SetLiteralNode)

    eq_comparison = label_name + Suppress("==") + string_literal
    eq_comparison.setParseAction(LabelToLiteralEqualityNode)

    not_eq_comparison = label_name + Suppress("!=") + string_literal
    not_eq_comparison.setParseAction(InequalityNode)

    in_comparison = label_name + Suppress(Keyword("in")) + set_literal
    in_comparison.setParseAction(LabelInSetLiteralNode)

    not_in = Suppress(Keyword("not") + Keyword("in"))
    not_in_comparison = label_name + not_in + set_literal
    not_in_comparison.setParseAction(NotInNode)

    has_check = (Suppress("has(") +
                 Word(LABEL_CHARS) +
                 Suppress(")"))
    has_check.setParseAction(HasNode)

    comparison = (eq_comparison |
                  not_eq_comparison |
                  in_comparison |
                  not_in_comparison |
                  has_check)

    paren_expr = (Suppress("(") + expr + Suppress(")"))

    value = comparison | paren_expr

    and_expr = value + ZeroOrMore(Suppress("&&") + value)
    and_expr.setParseAction(simplify_and_node)

    or_expr = and_expr + ZeroOrMore(Suppress("||") + and_expr)
    or_expr.setParseAction(simplify_or_node)

    expr << or_expr

    grammar = expr + StringEnd()
    return grammar
Ejemplo n.º 6
0
    def foldersByAnnotation(folderlist, type):
        result = []
        parse = QuotedString('"') + QuotedString('"') + Suppress("(") + Group(OneOrMore(QuotedString('"'))) + Suppress(")")

        for folder in folderlist:
            tmp = parse.parseString( folder )
            if type in tmp[2][1]:
                result.append(tmp[0])

        return result
Ejemplo n.º 7
0
    def _getPattern(self):
        arith_expr = Forward()
        comp_expr = Forward()
        logic_expr = Forward()
        LPAR, RPAR, SEMI = map(Suppress, "();")
        identifier = Word(alphas+"_", alphanums+"_")
        multop = oneOf('* /')
        plusop = oneOf('+ -')
        expop = Literal( "^" )
        compop = oneOf('> < >= <= != ==')
        andop = Literal("AND")
        orop = Literal("OR")
        current_value = Literal( "." )
        assign = Literal( "=" )
        # notop = Literal('NOT')
        function = oneOf(' '.join(self.FUNCTIONS))
        function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR)
        aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
        single_column = QuotedString(quoteChar='[', endQuoteChar=']')
        integer = Regex(r"-?\d+")
        real = Regex(r"-?\d+\.\d*")

        # quotedString enables strings without quotes to pass

        operand = \
            function_call.setParseAction(self.__evalFunction) | \
            aggregate_column.setParseAction(self.__evalAggregateColumn) | \
            single_column.setParseAction(self.__evalSingleColumn) | \
            ((real | integer).setParseAction(self.__evalConstant)) | \
            quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
            current_value.setParseAction(self.__evalCurrentValue) | \
            identifier.setParseAction(self.__evalString)

        arith_expr << operatorPrecedence(operand,
            [
             (expop, 2, opAssoc.LEFT, self.__expOp),
             (multop, 2, opAssoc.LEFT, self.__multOp),
             (plusop, 2, opAssoc.LEFT, self.__addOp),
            ])

        # comp_expr = Group(arith_expr + compop + arith_expr)
        comp_expr << operatorPrecedence(arith_expr,
            [
                (compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
            ])

        logic_expr << operatorPrecedence(comp_expr,
            [
                (andop, 2, opAssoc.LEFT, self.__evalLogicOp),
                (orop, 2, opAssoc.LEFT, self.__evalLogicOp)
            ])

        pattern = logic_expr + StringEnd()
        return pattern
Ejemplo n.º 8
0
def transform_human(text):
    """Transform user input into something Script can read."""
    # these are parseActions for pyparsing.
    def str_literal_to_hex(s, loc, toks):
        for i, t in enumerate(toks):
            toks[i] = ''.join(['0x', t.encode('hex')])
        return toks
    # ^ parseActions for pyparsing end here.
    str_literal = QuotedString('"')
    str_literal.setParseAction(str_literal_to_hex)

    transformer = ZeroOrMore( str_literal )
    return transformer.transformString(text)
Ejemplo n.º 9
0
 def calcValue(self,testCaseValues=None):
     if not self.containsExternalCode:
         #evaluate python code
         try:
             compiledExpression=compile(self.code.strip(),'<output-check>','eval')
             value=eval(compiledExpression,testCaseValues)
         except Exception:
             raise SyntaxSemanticException("Invalid Syntax in %s"%(self.code))
         self.value=str(value)
     else:
         def repl(x):
             return "%" if x[0]=="" else str(testCaseValues[x[0]]) 
         var=QuotedString("%").setParseAction(repl)
         self.value=var.transformString(self.code)
Ejemplo n.º 10
0
    def _build_grammar(self):
        expr = Forward()

        float_lit = Combine(Word(nums) + '.' + Word(nums))
        float_lit.setName('float')
        float_lit.setParseAction(lambda x: \
                                     self.to_literal(float(x[0])))

        int_lit = Word(nums)
        int_lit.setName('int')
        int_lit.setParseAction(lambda x: \
                                   self.to_literal(int(x[0])))

        num = (float_lit | int_lit)
        num.setParseAction(lambda x: x[0])

        tag_name = Word(alphas + "_", alphanums + "_")
        tag_name.setName('tag_name')
        tag_name.setParseAction(lambda t: tag_reference.TagReference(t[0]))

        quoted_string = QuotedString("'")
        quoted_string.setParseAction(lambda s: self.to_literal(s[0]))

        oper = oneOf('+ * / -')
        oper.setParseAction(lambda o: o[0])

        lpar  = Literal("(").suppress()
        rpar  = Literal(")").suppress()

        arith = Group(lpar + expr + oper + expr + rpar)
        arith.setParseAction(lambda t: \
                                 self.to_arith(t[0][0], t[0][1], t[0][2]))

        assign = tag_name + '=' + expr
        assign.setName('assign')
        assign.setParseAction(lambda x: self.to_assign(x[0],x[2]))

        print_tags = Literal('?')
        print_tags.setParseAction(lambda x: self.to_print_tags())

        expr <<(arith|assign|tag_name|num|quoted_string|print_tags)
        expr.setParseAction(lambda x: x[0])
        return expr
Ejemplo n.º 11
0
def _router_grammer():
    """Make the url route grammer"""
    wildcard = Literal('*')
    path_separator = Literal('/').setParseAction(lambda t: ('sep', t))
    extract = QuotedString(
        quoteChar='{',
        endQuoteChar='}').setParseAction(lambda t: ('extract', t))
    static = Word(''.join(
        c for c in printables
        if c not in '?{}/')).setParseAction(lambda t: ('static', t))
    route_parse = OneOrMore(extract | static | path_separator | wildcard)
    return route_parse
Ejemplo n.º 12
0
class NginxParser(object):
    """
    A class that parses nginx configuration with pyparsing
    """

    # constants

    left_bracket = Literal("{").suppress()
    right_bracket = Literal("}").suppress()
    semicolon = Literal(";").suppress()

    key = Word(alphanums + "_/")
    value = QuotedString(quoteChar='"', escChar='\\') \
          | QuotedString(quoteChar="'", escChar='\\') \
          | Word(alphanums + "*_/.-+$:=")
    assignment = (key + Group(ZeroOrMore(value)) + semicolon)
    block = Forward()
    subblock = Forward()

    subblock << ZeroOrMore(Group(assignment) | block)

    block << Group(key + SkipTo('{') + left_bracket + Group(subblock) +
                   right_bracket)

    script = OneOrMore(Group(assignment) | block).ignore(pythonStyleComment)

    def __init__(self, source):
        self.source = source

    def parse(self):
        """
        Returns the parsed tree.
        """
        return self.script.parseString(self.source)

    def as_list(self):
        """
        Returns the list of tree.
        """
        return self.parse().asList()
Ejemplo n.º 13
0
def build_parser():
    """
    Build a pyparsing parser for our custom topology description language.

    :return: A pyparsing parser.
    :rtype: pyparsing.MatchFirst
    """
    ParserElement.setDefaultWhitespaceChars(' \t')
    nl = Suppress(LineEnd())
    inumber = Word(nums).setParseAction(lambda l, s, t: int(t[0]))
    fnumber = (Combine(
        Optional('-') + Word(nums) + '.' + Word(nums) +
        Optional('E' | 'e' + Optional('-') + Word(nums)))
               ).setParseAction(lambda toks: float(toks[0]))
    boolean = (CaselessLiteral('true') | CaselessLiteral('false')
               ).setParseAction(lambda l, s, t: t[0].casefold() == 'true')
    comment = Literal('#') + restOfLine + nl
    text = QuotedString('"')
    identifier = Word(alphas, alphanums + '_')
    empty_line = LineStart() + LineEnd()
    item_list = ((text | fnumber | inumber | boolean) +
                 Optional(Suppress(',')) + Optional(nl))
    custom_list = (Suppress('(') + Optional(nl) + Group(OneOrMore(item_list)) +
                   Optional(nl) +
                   Suppress(')')).setParseAction(lambda tok: tok.asList())
    attribute = Group(
        identifier('key') + Suppress(Literal('=')) +
        (custom_list | text | fnumber | inumber | boolean
         | identifier)('value') + Optional(nl))
    attributes = (Suppress(Literal('[')) + Optional(nl) +
                  OneOrMore(attribute) + Suppress(Literal(']')))

    node = identifier('node')
    port = Group(node + Suppress(Literal(':')) +
                 (identifier | inumber)('port'))
    link = Group(
        port('endpoint_a') + Suppress(Literal('--')) + port('endpoint_b'))

    environment_spec = (attributes + nl).setResultsName('env_spec',
                                                        listAllMatches=True)
    nodes_spec = (Group(
        Optional(attributes)('attributes') + Group(OneOrMore(node))('nodes')) +
                  nl).setResultsName('node_spec', listAllMatches=True)
    ports_spec = (Group(
        Optional(attributes)('attributes') + Group(OneOrMore(port))('ports')) +
                  nl).setResultsName('port_spec', listAllMatches=True)
    link_spec = (Group(Optional(attributes)('attributes') + link('links')) +
                 nl).setResultsName('link_spec', listAllMatches=True)

    statements = OneOrMore(comment | link_spec | ports_spec | nodes_spec
                           | environment_spec | empty_line)
    return statements
Ejemplo n.º 14
0
    def _create_grammar(self):
        """
        Pyparsing implementation of a where clause grammar based on http://pyparsing.wikispaces.com/file/view/simpleSQL.py

        The query language is a series of statements separated by AND or OR operators and parentheses can be used to group/provide
        precedence.

        A statement is a combination of three strings "<filter> <operator> <value>" or "<filter> <operator> <filter>".

        A value can be a string, integer or a real(floating) number or a (ISO YYYY-MM-DD) date.

        An operator must be one of "= != < > >= <= !:" and are translated into django __lte or equivalent suffixes.
        See self.as_q

        Example
        something < 10 AND other >= 2015-01-01 AND (foo < 1 OR bar > 1)

        """
        quoted_string_excluding_quotes = QuotedString(
            '"',
            escChar='\\').setParseAction(lambda token: StringValue(token[0]))
        and_ = Keyword('and', caseless=True)
        or_ = Keyword('or', caseless=True)
        binary_op = oneOf('=> =< = < > >= <= : != !:',
                          caseless=True).setResultsName('operator')

        # define query tokens
        identifier = Word(alphas, alphanums + '_$-.').setName('identifier')
        raw_value_chars = alphanums + '_$-+/$%*;?@[]\\^`{}|~.'
        raw_value = Word(raw_value_chars, raw_value_chars).setName('raw_value')
        value_string = quoted_string_excluding_quotes | raw_value

        # Define a where expression
        where_expression = Forward()
        binary_operator_statement = (identifier + binary_op +
                                     value_string).setParseAction(
                                         self._binary_op_to_q)
        unary_operator_statement = (identifier |
                                    (Char('!') + identifier)).setParseAction(
                                        self._unary_op_to_q)
        free_text_statement = quotedString.copy().setParseAction(
            self._freetext_to_q)
        operator_statement = binary_operator_statement | free_text_statement | unary_operator_statement
        where_condition = Group(operator_statement
                                | ('(' + where_expression + ')'))
        where_expression << where_condition + ZeroOrMore(
            (and_ | or_) + where_expression)

        # define the full grammar
        query_statement = Forward()
        query_statement << Group(where_expression).setResultsName("where")
        return query_statement
Ejemplo n.º 15
0
 def parse_payload(self,payload):
     expr = Forward()
     
     LPAR, RPAR, SEMI = map(Suppress, "();")
     identifier = Word(alphas+"_", alphanums+"_")
     function_call = identifier.setResultsName("name") + LPAR + Group(Optional(delimitedList(expr))) + RPAR
     integer = Regex(r"-?\d+")
     real = Regex(r"-?\d+\.\d*")
     
     qstr = QuotedString(quoteChar = '"', escChar = '\\', unquoteResults=False)
     qstrsingle = QuotedString(quoteChar = "'", escChar = '\\', unquoteResults=False)
     operand = (identifier | real | integer | qstr | qstrsingle )
     plusop = oneOf('+ -')
     expr << infixNotation( operand, [ (plusop, 2, opAssoc.LEFT) ])
     out = []
     for t,s,e in function_call.scanString( payload ):
             out.append({
                             "action" : t[0],
                             "arguments" : t[1].asList() if type(t[1])!=str else t[1] 
                        }
                       )
     return out
Ejemplo n.º 16
0
def parse_stats(fin):
    from pyparsing import (LineEnd, Forward, Word, alphas, QuotedString, Group,
                           OneOrMOre)

    EOL = LineEnd().suppress()

    new_stmt = Forward().setName('new statement')
    new_stmt << (Word('new').suppress() + Word(alphas) +
                 QuotedString('"')) + EOL

    data_stmt = Forward().setName('data statement')
    data_stmt << (Word('data').suppress() + QuotedString('"') +
                  QuotedString('"')) + EOL

    p = OneOrMore(Group(new_stmt + Group(OneOrMore(data_stmt))))

    results = p.parseFile(fin)

    for row in results:
        action, name, data = row
        data = dict(chunks(data, 2))
        yield (action, name, data)
Ejemplo n.º 17
0
def parse_search_query(query):

    unicode_printables = u''.join(
        unichr(c) for c in xrange(65536) if not unichr(c).isspace())
    word = TextNode.group(Word(unicode_printables))
    exact = ExactNode.group(
        QuotedString('"', unquoteResults=True, escChar='\\'))
    term = exact | word
    comparison_name = Word(unicode_printables, excludeChars=':')
    comparison = ComparisonNode.group(comparison_name + Literal(':') + term)
    content = OneOrMore(comparison | term)

    return content.parseString(query)
Ejemplo n.º 18
0
def d3dmdf2dict(mdf_file):
    d = {}
    parameter = Word(alphas)
    # floatNumber = Regex(r'\d+(\.\d*)?([eE]\+\d+)?')
    floatNumber = Regex(r'-?\d+(\.\d*)?([eE][\+-]\d+)?').setParseAction(lambda s,l,t: [ float(t[0]) ] )
    d3dline = parameter + Suppress('=') + Optional(
        OneOrMore(QuotedString("#", unquoteResults=False)) | OneOrMore(floatNumber))
    for param in d3dline.searchString(file(mdf_file).read()):
        d[param[0].lower()] = ''.join(param[1:])
        # remove comments
    del d['commnt']
    print d
    return d
Ejemplo n.º 19
0
    def __init__(self):
        # speed up infixNotation considerably at the price of some cache memory
        ParserElement.enablePackrat()

        boolean = Keyword('True') | Keyword('False')
        none = Keyword('None')
        integer = Word(nums)
        real = Combine(Word(nums) + "." + Word(nums))
        string = (QuotedString('"', escChar='\\')
                  | QuotedString("'", escChar='\\'))
        regex = QuotedString('/', escChar='\\')
        identifier = Word(alphas, alphanums + '_')
        dereference = infixNotation(identifier, [
            (Literal('.'), 2, opAssoc.LEFT, EvalArith),
        ])
        result = (Keyword('bad') | Keyword('fail') | Keyword('good')
                  | Keyword('ignore') | Keyword('unknown'))
        rval = boolean | none | real | integer | string | regex | result | dereference
        rvallist = Group(
            Suppress('[') + Optional(delimitedList(rval)) + Suppress(']'))
        rvalset = Group(
            Suppress('{') + Optional(delimitedList(rval)) + Suppress('}'))
        operand = rval | rvallist | rvalset

        # parse actions replace the parsed tokens with an instantiated object
        # which we can later call into for evaluation of its content
        boolean.setParseAction(EvalBoolean)
        none.setParseAction(EvalNone)
        integer.setParseAction(EvalInteger)
        real.setParseAction(EvalReal)
        string.setParseAction(EvalString)
        regex.setParseAction(EvalRegex)
        identifier.setParseAction(EvalIdentifier)
        result.setParseAction(EvalResult)
        rvallist.setParseAction(EvalList)
        rvalset.setParseAction(EvalSet)

        identity_test = Keyword('is') + ~Keyword('not') | Combine(
            Keyword('is') + Keyword('not'), adjacent=False, joinString=' ')
        membership_test = Keyword('in') | Combine(
            Keyword('not') + Keyword('in'), adjacent=False, joinString=' ')
        comparison_op = oneOf('< <= > >= != == isdisjoint')
        comparison = identity_test | membership_test | comparison_op

        self.parser = infixNotation(operand, [
            (Literal('**'), 2, opAssoc.LEFT, EvalPower),
            (oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier),
            (oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith),
            (oneOf('+ -'), 2, opAssoc.LEFT, EvalArith),
            (oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith),
            (Literal('&'), 2, opAssoc.LEFT, EvalArith),
            (Literal('^'), 2, opAssoc.LEFT, EvalArith),
            (Literal('|'), 2, opAssoc.LEFT, EvalArith),
            (comparison, 2, opAssoc.LEFT, EvalLogic),
            (Keyword('not'), 1, opAssoc.RIGHT, EvalModifier),
            (Keyword('and'), 2, opAssoc.LEFT, EvalLogic),
            (Keyword('or'), 2, opAssoc.LEFT, EvalLogic),
            (Keyword('->'), 2, opAssoc.LEFT, EvalArith),
        ])
Ejemplo n.º 20
0
    def reInit(self):
        # Since this class gets pickled in ActionWindow, the class is never 'destroyed'
        # Since, a functions docstring wouldn't be reflected back to puddletag
        # if it were changed calling this function to 're-read' it is a good idea.
        if not self.function.__doc__:
            return
        self.doc = self.function.__doc__.split("\n")

        identifier = QuotedString('"') | Combine(
            Word(alphanums + ' !"#$%&\'()*+-./:;<=>?@[\\]^_`{|}~'))
        tags = delimitedList(identifier)

        self.info = [z for z in tags.parseString(self.doc[0])]
Ejemplo n.º 21
0
    def __init__(self, comm_file_path):
        expression_spaced = Forward()
        expression = Forward()
        args_spaced = Forward()

        cb = Optional(',') + ')'  # closing_brackets might include a ','
        ob = Optional(' ') + '(' + Optional(
            ' ')  # closing_brackets might include a ' '

        value = (Or([
            pyparsing_common.identifier.copy().setResultsName('id'),
            pyparsing_common.number.copy().setResultsName('number'),
            QuotedString("'").setResultsName('string')
        ])).setParseAction(Value).setResultsName('value')

        values = (ZeroOrMore(
            value.setResultsName('valueList', listAllMatches=True) +
            Optional(','))).setParseAction(Values)

        keyword = pyparsing_common.identifier.copy()

        keyword_argument = (keyword.setResultsName('keyword') + '=' +
                            expression_spaced.setResultsName('expression')
                            ).setParseAction(Keyword_argument)

        keyword_arguments = (keyword_argument.setResultsName(
            'keyword_argument', listAllMatches=True) +
                             ZeroOrMore(',' + keyword_argument.setResultsName(
                                 'keyword_argument', listAllMatches=True))
                             ).setParseAction(Keyword_arguments)

        expression << (Or([
            value, (ob + values.setResultsName('values') + cb), '_F' + ob +
            keyword_arguments.setResultsName('keyword_arguments') + cb,
            ob + expression.setResultsName('expression') + cb
        ])).setParseAction(Expression)

        expression_spaced << (Or([expression, ob + expression_spaced + cb]))

        left_side = pyparsing_common.identifier.setResultsName('left_side')
        operator_name = pyparsing_common.identifier.setResultsName(
            'operator_name')
        paragraph = (
            Optional(left_side + "=") + operator_name + ob +
            Optional(keyword_arguments.setResultsName('keyword_arguments')) +
            cb + Optional(';')).setParseAction(Paragraph)

        file = OneOrMore(paragraph).setResultsName(
            'paragraphs').setParseAction(File)

        self.beam_data_model = file.parseFile(comm_file_path)
Ejemplo n.º 22
0
 def parse(self, config):
     VALUE = OneOrMore(Word(alphanums + "-/.:_+[],") | QuotedString('"'))
     context = []
     inactive_level = 1
     for line in config.splitlines():
         line = line.rstrip()
         if "no " in line:
             line = line.replace("no ", "")
         if not line:
             continue
         elif line.startswith(" " * inactive_level):
             inactive_level = len(context) + 1
             context += [line.split()[0].strip()]
         elif "set" in line:
             inactive_level = len(context) + 1
         elif "exit" in line:
             context.pop(-1)
             inactive_level = len(context) - 1
         elif "!" in line and context:
             context = []
             inactive_level = inactive_level - 1
             # continue
             # if inactive_level is not None and inactive_level >= len(context):
             #    inactive_level = None
         elif "!" in line:
             continue
         elif context and len(context) >= inactive_level:
             if context[-1] not in line:
                 context.pop(-1)
                 context += [line.split()[0].strip()]
         else:
             context += [line.split()[0].strip()]
         cp = " ".join(context).split() + line.split()
         h = self.handlers
         # print cp
         for p in cp:
             if p in h:
                 h = h[p]
             elif "*" in h:
                 h = h["*"]
             else:
                 break
             if callable(h):
                 h(self, VALUE.parseString(" ".join(cp)))
                 break
             elif h is None:
                 break
     # Yield facts
     for f in self.iter_facts():
         yield f
Ejemplo n.º 23
0
def get_grammar():
    assign = Literal('=')
    lpar = Literal('{').suppress()
    rpar = Literal('}').suppress()
    vartype = Word(alphas + '_')
    varname = Word(alphas + nums + '._')
    varvalue = (Regex(r'0x[0-9a-fA-F]{2,}') | \
                Word(nums) | \
                QuotedString("\"\"\"", multiline=True))
    expr = Forward()
    atomic = vartype + varname + assign + varvalue
    struct = vartype + varname + assign + Group(lpar + expr + rpar)
    expr << ZeroOrMore(atomic | struct)
    return expr
Ejemplo n.º 24
0
def _parse_data(data: str) -> List[_PackageData]:
    lpar, rpar, lbrk, rbrk, dot = map(Suppress, '()[].')
    nil = Suppress('nil')

    pkgname = Word(printables)
    decimal = Regex(r'0|-?[1-9]\d*').setParseAction(lambda t: int(t[0]))
    qstring = QuotedString(quoteChar='"', escChar='\\')

    version = (lpar + OneOrMore(decimal) +
               rpar).setParseAction(lambda s, l, t: ['.'.join(map(str, t))])

    dependency_entry = lpar + pkgname + version + rpar
    dependency_list = ((lpar + OneOrMore(dependency_entry) + rpar) | nil)

    people_list = OneOrMore(qstring | dot | nil)

    keyval_url = (lpar + (Suppress(':url') | Suppress(':homepage')) + dot +
                  qstring +
                  rpar).setParseAction(lambda s, l, t: [('url', t[0])])
    keyval_keywords = (lpar + Suppress(':keywords') + ZeroOrMore(qstring) +
                       rpar).setParseAction(
                           lambda s, l, t: [('keywords', [str(k) for k in t])])
    keyval_commit = (lpar + Suppress(':commit') + dot + qstring +
                     rpar).setParseAction(lambda s, l, t: [('commit', t[0])])
    keyval_maintainer = (
        lpar + Suppress(':maintainer') + people_list + rpar
    ).setParseAction(lambda s, l, t: [('maintainer', [str(m) for m in t])])
    keyval_author = (
        lpar + Suppress(':author') + people_list +
        rpar).setParseAction(lambda s, l, t: [('author', [str(a) for a in t])])
    keyval_authors = (lpar + Suppress(':authors') +
                      OneOrMore(lpar + people_list + rpar) +
                      rpar).setParseAction(
                          lambda s, l, t: [('authors', [str(a) for a in t])])

    keyval_item = keyval_url | keyval_keywords | keyval_commit | keyval_maintainer | keyval_authors | keyval_author

    keyvals = (lpar + ZeroOrMore(keyval_item) + rpar
               ).setParseAction(lambda s, l, t: [{k: v
                                                  for k, v in t}]
                                ) | nil.setParseAction(lambda s, l, t: [{}])

    package_entry = (lpar + pkgname + dot + lbrk + version +
                     Suppress(dependency_list) + qstring +
                     Suppress(Word(alphas)) + keyvals + rbrk +
                     rpar).setParseAction(lambda s, l, t: [_PackageData(*t)])

    root = lpar + Suppress(decimal) + ZeroOrMore(package_entry) + rpar

    return root.parseString(data, parseAll=True)  # type: ignore
Ejemplo n.º 25
0
def _define_grammar():
    """
    Creates and returns a copy of the selector grammar.

    Wrapped in a function to avoid polluting the module namespace.
    """
    expr = Forward()

    label_name = Word(LABEL_CHARS)
    label_name.setParseAction(LabelNode)

    string_literal = QuotedString('"') | QuotedString("'")
    string_literal.setParseAction(LiteralNode)

    set_literal = (Suppress("{") +
                   delimitedList(QuotedString('"') | QuotedString("'"), ",") +
                   Suppress("}"))
    set_literal.setParseAction(SetLiteralNode)

    eq_comparison = label_name + Suppress("==") + string_literal
    eq_comparison.setParseAction(LabelToLiteralEqualityNode)

    not_eq_comparison = label_name + Suppress("!=") + string_literal
    not_eq_comparison.setParseAction(InequalityNode)

    in_comparison = label_name + Suppress(Keyword("in")) + set_literal
    in_comparison.setParseAction(LabelInSetLiteralNode)

    not_in = Suppress(Keyword("not") + Keyword("in"))
    not_in_comparison = label_name + not_in + set_literal
    not_in_comparison.setParseAction(NotInNode)

    has_check = (Suppress("has(") + Word(LABEL_CHARS) + Suppress(")"))
    has_check.setParseAction(HasNode)

    comparison = (eq_comparison | not_eq_comparison | in_comparison
                  | not_in_comparison | has_check)

    paren_expr = (Suppress("(") + expr + Suppress(")"))

    value = comparison | paren_expr

    and_expr = value + ZeroOrMore(Suppress("&&") + value)
    and_expr.setParseAction(simplify_and_node)

    or_expr = and_expr + ZeroOrMore(Suppress("||") + and_expr)
    or_expr.setParseAction(simplify_or_node)

    expr << or_expr

    grammar = expr + StringEnd()
    return grammar
Ejemplo n.º 26
0
def grammar():
    parenthesis = Forward()
    parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")"
    parenthesis.setParseAction(join_string_act)

    quoted_string = "'" + OneOrMore(CharsNotIn("'")) + "'"
    quoted_string.setParseAction(join_string_act)

    quoted_default_value = "DEFAULT" + quoted_string + OneOrMore(
        CharsNotIn(", \n\t"))
    quoted_default_value.setParseAction(quoted_default_value_act)

    field_def = OneOrMore(quoted_default_value
                          | Word(alphanums + "_\"'`:-/[].") | parenthesis)
    field_def.setParseAction(Field)

    tablename_def = (Word(alphas + "`_.") | QuotedString("\""))

    field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def)

    create_table_def = Literal(
        "CREATE") + "TABLE" + tablename_def.setResultsName(
            "tableName") + "(" + field_list_def.setResultsName(
                "fields") + ")" + ";"
    create_table_def.setParseAction(Table)

    add_fkey_def = Literal(
        "ALTER") + "TABLE" + "ONLY" + tablename_def.setResultsName(
            "tableName") + "ADD" + "CONSTRAINT" + Word(
                alphanums + "_") + "FOREIGN" + "KEY" + "(" + delimitedList(
                    Word(alphanums + "_")
                ).setResultsName("keyName") + ")" + "REFERENCES" + Word(
                    alphanums +
                    "._").setResultsName("fkTable") + "(" + delimitedList(
                        Word(alphanums + "_")
                    ).setResultsName("fkCol") + ")" + Optional(
                        Literal("DEFERRABLE")) + Optional(
                            Literal("ON") + "DELETE" +
                            (Literal("CASCADE") | Literal("RESTRICT"))) + ";"
    add_fkey_def.setParseAction(FKey)

    other_statement_def = OneOrMore(CharsNotIn(";")) + ";"
    other_statement_def.setParseAction(OtherStatement)

    comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
    comment_def.setParseAction(OtherStatement)

    return OneOrMore(comment_def | create_table_def | add_fkey_def
                     | other_statement_def)
Ejemplo n.º 27
0
def query_parser() -> ParserElement:
    # Creating the grammar
    valid_chars = printables.replace("(", "").replace(")", "").replace(
        ":", "").replace("\"", "")
    expr = Forward().setName("expression")

    quotes = QuotedString(
        '"', "\\").setName("quoted string").setResultsName("quotes")

    brackets = Group(Literal("(").suppress() + expr + Literal(")").suppress()) \
        .setName("bracketed expression").setResultsName("brackets")

    words = Word(valid_chars).setName("word").setResultsName("word")

    exception_elem = Group(quotes | words).setName("exception element")\
        .setResultsName("exception_element", listAllMatches=True)
    exception = Group(exception_elem | (
        Literal("(") + exception_elem +
        ZeroOrMore(pyparsing.Optional(CaselessKeyword("or")) +
                   exception_elem) +
        Literal(")"))).setName("exception").setResultsName("exception")
    exception_connector = (CaselessKeyword("except") | CaselessKeyword("ignore"))\
        .setName("Except").setResultsName("except")
    exception_word = words.setName("exception word").setResultsName(
        "exception_word")

    word_with_exception = Group(exception_word + exception_connector + exception) \
        .setName("word with exception").setResultsName("word_with_exception")
    word_with_exception_brackets = Literal(
        "(") + word_with_exception + Literal(")")

    field_name = Group((Literal("@").suppress() + Word(valid_chars)) | (Word(valid_chars) + Literal(":").suppress())) \
        .setName("field name").setResultsName("field_name")
    field_value = Group(quotes | word_with_exception_brackets | word_with_exception | words)\
        .setName("field value").setResultsName("field_value")
    field = Group(field_name +
                  field_value).setName("field").setResultsName("field")

    negator = Group(pyparsing.Optional(Literal("!") | Literal("-") | CaselessKeyword("not"))) \
        .setName("negator").setResultsName("negator")
    element = Group(quotes | brackets | field | word_with_exception | words) \
        .setName("element").setResultsName("element")
    full_element = Group(negator +
                         element).setName("full element").setResultsName(
                             "full_element", listAllMatches=True)
    connector = Group(pyparsing.Optional(CaselessKeyword("or") | CaselessKeyword("and"))) \
        .setName("connector").setResultsName("connector", listAllMatches=True)
    expr <<= full_element + ZeroOrMore(connector + full_element)
    return expr
Ejemplo n.º 28
0
 def compiled_sql(self):
     where_op = oneOf('> >= < <= = IN !=') | Literal("NOT IN")
     number = Word(nums)
     string = (QuotedString(quoteChar="'") | QuotedString(
         quoteChar='"')).setParseAction(lambda x: '"' + x[0] + '"')
     where_val = number | string
     lpar = Literal('(')
     rpar = Literal(')')
     tables = Group(
         Word(alphas) + ZeroOrMore(Suppress(Literal('>>>')) + Word(alphas)))
     tbl_col = tables + Suppress(Literal('.')) + Word(alphas)
     where_clause = tbl_col + where_op + where_val
     where_clause.setParseAction(lambda x: self._make_atomic(
         tables=x[0], col_name=x[1], op=x[2], value=x[3]))
     expr = Forward()
     atom = where_clause | (lpar + expr + rpar)
     op = Literal("&").setParseAction(
         replaceWith("INTERSECT")) | Literal("|").setParseAction(
             replaceWith("UNION"))
     expr << (atom + ZeroOrMore(op + expr)) | (
         Suppress("(") + atom + ZeroOrMore(op + expr) + Suppress(")"))
     results = expr.parseString(self.pre_query)
     print(' '.join(results))
     return results
Ejemplo n.º 29
0
    def _parse_query(self):
        """
        Defines and parses the Resource DSL based on the query associated with the PanoptesResourceDSL object

        Returns:
            list: The list of tokens parsed

        Raises:
            ParseException: This exception is raised if any parsing error occurs
        """
        resource_fields = oneOf(
            'resource_site resource_class resource_subclass resource_type resource_id resource_endpoint',
            caseless=True)
        resource_metadata = CaselessLiteral('resource_metadata') + Literal(
            '.') + Word(alphanums + '_')

        and_ = CaselessKeyword('AND').setParseAction(upcaseTokens)
        or_ = CaselessKeyword('OR').setParseAction(upcaseTokens)
        not_ = CaselessKeyword('NOT').setParseAction(upcaseTokens)
        in_ = CaselessKeyword('IN').setParseAction(upcaseTokens)
        like_ = CaselessKeyword('LIKE').setParseAction(upcaseTokens)

        operators = oneOf("= != eq ne",
                          caseless=True).setParseAction(upcaseTokens)

        query_expression = Forward()

        query_l_val = (resource_fields
                       | resource_metadata).setParseAction(downcaseTokens)
        query_r_val = QuotedString(quoteChar='"', escChar='\\')

        query_condition = Group((query_l_val + operators + query_r_val)
                                | (query_l_val + Optional(not_) + like_ +
                                   query_r_val)
                                | (query_l_val + Optional(not_) + in_ + '(' +
                                   delimitedList(query_r_val) + ')'))

        query_expression << query_condition - ZeroOrMore(
            (and_ | or_) - query_condition)

        try:
            tokens = query_expression.parseString(self._query, parseAll=True)
        except ParseException as e:
            raise e

        return tokens
Ejemplo n.º 30
0
def create_names_db():
    p = Suppress(SkipTo('"')) + QuotedString('"') + Suppress(SkipTo("."))

    with open("persondata_en.ttl") as pd:
        for line in pd:
            surname, givenname = None, None
            if "/name" in line:
                name = p.parseString(line)[0]
                line = pd.__next__()
                if "/surname" in line:
                    surname = p.parseString(line)[0]
                    line = pd.__next__()
                    if "/givenName" in line:
                        givenname = p.parseString(line)[0]
                n = Name(name=name, surname=surname, givenname=givenname)
                session.add(n)
                session.commit()
Ejemplo n.º 31
0
    def _getPattern(self):
        arith_expr = Forward()
        comp_expr = Forward()
        logic_expr = Forward()
        LPAR, RPAR, SEMI = map(Suppress, "();")
        identifier = Word(alphas + "_", alphanums + "_")
        multop = oneOf('* /')
        plusop = oneOf('+ -')
        expop = Literal("^")
        compop = oneOf('> < >= <= != ==')
        andop = Literal("AND")
        orop = Literal("OR")
        current_value = Literal(".")
        assign = Literal("=")
        # notop = Literal('NOT')
        function = oneOf(' '.join(self.FUNCTIONS))
        function_call = Group(
            function.setResultsName('fn') + LPAR +
            Optional(delimitedList(arith_expr)) + RPAR)
        aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
        single_column = QuotedString(quoteChar='[', endQuoteChar=']')
        integer = Regex(r"-?\d+")
        real = Regex(r"-?\d+\.\d*")

        # quotedString enables strings without quotes to pass

        operand = \
            function_call.setParseAction(self.__evalFunction) | \
            aggregate_column.setParseAction(self.__evalAggregateColumn) | \
            single_column.setParseAction(self.__evalSingleColumn) | \
            ((real | integer).setParseAction(self.__evalConstant)) | \
            quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
            current_value.setParseAction(self.__evalCurrentValue) | \
            identifier.setParseAction(self.__evalString)

        arith_expr << operatorPrecedence(operand, [
            (expop, 2, opAssoc.LEFT, self.__expOp),
            (multop, 2, opAssoc.LEFT, self.__multOp),
            (plusop, 2, opAssoc.LEFT, self.__addOp),
        ])

        # comp_expr = Group(arith_expr + compop + arith_expr)
        comp_expr << operatorPrecedence(arith_expr, [
            (compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
        ])

        logic_expr << operatorPrecedence(
            comp_expr, [(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
                        (orop, 2, opAssoc.LEFT, self.__evalLogicOp)])

        pattern = logic_expr + StringEnd()
        return pattern
Ejemplo n.º 32
0
def _parse_gitconfig(config: str) -> Dict[str, Any]:
    # Header
    header = Word(alphas) + Optional(QuotedString('"'))
    full_header = Suppress(LineStart()) + \
        nestedExpr(opener="[", closer="]", content=header) + \
        Suppress(LineEnd())
    #full_header = Suppress(LineStart()) + Suppress(Literal("[")) + \
    #    header + Suppress(Literal("]")) + Suppress(LineEnd())

    # Keys
    key = Word(alphas) + Suppress(Literal("=")) + Suppress(Optional(" ")) + \
        restOfLine()

    # Full pattern
    full_pattern = full_header + ZeroOrMore(key)

    #return full_header
    return [match for match in full_pattern.scanString(config)]
Ejemplo n.º 33
0
 def parse(self, config):
     VALUE = OneOrMore(Word(alphanums + "-/.:_+") | QuotedString("'"))
     context = []
     indent = []
     for l in config.splitlines():
         ls = l.strip()
         if not ls or ls.startswith("!"):
             continue  # Comment line
         match = self.rx_indent.search(l)
         ilevel = len(match.group(1))
         if not indent:
             indent = [ilevel]
             context = [ls]
         elif indent[-1] == ilevel:
             # Same level context
             context = context[:-1] + [ls]
         elif indent[-1] < ilevel:
             # Down
             context += [ls]
             indent += [ilevel]
         else:
             # Up
             while indent and indent[-1] >= ilevel:
                 indent.pop(-1)
                 context.pop(-1)
             context += [ls]
             indent += [ilevel]
         cp = " ".join(context).split()
         h = self.handlers
         for p in cp:
             if p in h:
                 h = h[p]
             elif "*" in h:
                 h = h["*"]
             else:
                 break
             if callable(h):
                 h(self, VALUE.parseString(" ".join(cp)))
                 break
             elif h is None:
                 break
     # Yield facts
     for f in self.iter_facts():
         yield f
Ejemplo n.º 34
0
 def parse(self, config):
     VALUE = OneOrMore(Word(alphanums + "-/.:_+[]") | QuotedString('"'))
     context = []
     inactive_level = None
     for line in config.splitlines():
         # @todo: Skip inactive blocks
         if "##" in line:
             line = line.split("##", 1)[0]
         line = line.strip()
         if line == "{master}":
             continue
         elif line.startswith("inactive:"):
             if line.endswith("{"):
                 inactive_level = len(context)
                 context += [line[9:-1].strip()]
         elif line.startswith("/*"):
             continue
         elif line.endswith("{"):
             context += [line[:-1].strip()]
         elif line == "}":
             context.pop(-1)
             if inactive_level is not None and inactive_level >= len(
                     context):
                 inactive_level = None
         elif line.endswith(";"):
             if inactive_level is not None:
                 continue
             cp = " ".join(context).split() + line[:-1].split()
             h = self.handlers
             for p in cp:
                 if p in h:
                     h = h[p]
                 elif "*" in h:
                     h = h["*"]
                 else:
                     break
                 if callable(h):
                     h(self, VALUE.parseString(" ".join(cp)))
                     break
                 elif h is None:
                     break
     # Yield facts
     for f in self.iter_facts():
         yield f
Ejemplo n.º 35
0
def parse_treesblock(infile):
    import string
    from pyparsing import Optional, Word, Regex, CaselessKeyword, Suppress
    from pyparsing import QuotedString
    comment = Optional(Suppress("[&") + Regex(r'[^]]+') + Suppress("]"))
    name = Word(alphanums + "_") | QuotedString("'")
    newick = Regex(r'[^;]+;')
    tree = (CaselessKeyword("tree").suppress() + Optional("*").suppress() +
            name.setResultsName("tree_name") +
            comment.setResultsName("tree_comment") + Suppress("=") +
            comment.setResultsName("root_comment") +
            newick.setResultsName("newick"))

    ## treesblock = Group(beginblock +
    ##                    Optional(ttable.setResultsName("ttable")) +
    ##                    Group(OneOrMore(tree)) +
    ##                    endblock)

    def parse_ttable(f):
        ttable = {}
        while True:
            s = f.next().strip()
            if s.lower() == ";":
                break
            if s[-1] in ",;":
                s = s[:-1]
            k, v = s.split()
            ttable[k] = v
            if s[-1] == ";":
                break
        return ttable

    ttable = {}
    while True:
        try:
            s = infile.next().strip()
        except StopIteration:
            break
        if s.lower() == "translate":
            ttable = parse_ttable(infile)
            # print("ttable: %s" % len(ttable))
        else:
            match = tree.parseString(s)
            yield Newick(match, ttable)
Ejemplo n.º 36
0
    def translate(self, text, filename):
        self.source = text
        self.super = None
        self.inheritance = 0
        self.declaration_lines = ['inheritance = 0']
        self.block_lines = []
        self.body_lines = ['def body():']

        self.target_lines = self.body_lines
        self.indent = 1

        template_close = Literal('%>')
        white = White()

        attribute = Word(alphanums + '_') + Literal('=') + QuotedString(
            '"') + Optional(white)

        directive = "<%@" + Optional(white) + Word(
            alphanums + '_') + white + ZeroOrMore(attribute) + template_close
        declaration = "<%!" + SkipTo(template_close) + template_close
        expression = "<%=" + SkipTo(template_close) + template_close
        scriptlet = '<%' + SkipTo(template_close) + template_close

        template_text = directive | declaration | expression | scriptlet
        plain_text = Regex(r'((?!<%).|\s)+', re.MULTILINE)

        body = template_text | plain_text
        lit = OneOrMore(body)

        directive.setParseAction(self.compile_directive)
        declaration.setParseAction(self.compile_declaration)
        expression.setParseAction(self.compile_expression)
        scriptlet.setParseAction(self.compile_scriptlet)
        plain_text.setParseAction(self.compile_plain_text)

        lit.leaveWhitespace()
        lit.parseString(self.source)

        translated = '\n' + '\n'.join(self.declaration_lines + ['\n'] +
                                      self.block_lines + ['\n'] +
                                      self.body_lines)
        if self.super:
            translated = self.super.module_source + translated
        return translated
Ejemplo n.º 37
0
def __build_grammar():
    expr = Forward()

    k_select = CaselessLiteral("SELECT")
    k_from = CaselessLiteral("FROM")
    k_where = CaselessLiteral("WHERE")
    k_and = CaselessLiteral("AND")
    k_instances = CaselessLiteral("INSTANCES")
    qs = QuotedString("'", escQuote="''")

    identifier = Combine(
        Word(alphas + "_", exact=1) +
        Optional(Word(nums + alphas + "_")))("identifier")
    navigation = Group(identifier +
                       ZeroOrMore(Suppress(".") + identifier))("navigation")

    filter_predicate = Group(navigation + Suppress("=") +
                             (qs('value') | (Suppress('(') + expr('subquery') +
                                             Suppress(')'))))('predicate')
    where_clause = Group(
        Suppress(k_where) + filter_predicate +
        ZeroOrMore(Suppress(k_and) + filter_predicate))('where')

    # Pre filters
    impl = Optional(Suppress(CaselessLiteral("implementation"))) + qs('impl')
    cic = Suppress(CaselessLiteral("offer")) + qs('cic')
    lc = Suppress(CaselessLiteral("lc")) + qs('lc')
    envt = Suppress(CaselessLiteral("environment")) + qs('envt')
    pre_filter = Optional(envt) + Optional(lc) + Optional(cic) + Optional(
        impl) + FollowedBy(k_instances)

    # Dict query (only select some elements and navigate)
    nl_expr = Group(navigation + ZeroOrMore(Suppress(',') + navigation) +
                    FollowedBy(k_from))('selector')

    # The sum of all fears
    select = Group(
        Suppress(k_select) + Optional(nl_expr + Suppress(k_from)) +
        pre_filter + Suppress(k_instances) + Optional(where_clause) +
        Optional(CaselessLiteral('WITH COMPUTATIONS')('compute')))('select')

    expr << select
    return expr
Ejemplo n.º 38
0
def _create_filter_parser():
    and_kw = Keyword('AND')
    or_kw = Keyword('OR')
    variable = Literal('?') + Word(alphanums + '_').leaveWhitespace()
    uri_term = NotAny(Literal('"')) + Word(printables, excludeChars='>*')
    uri_part = Keyword('*') ^ uri_term ^ variable
    literal_term = QuotedString(quoteChar='"', escChar='\\')
    triple = Group(
        Literal('<').suppress() + uri_part.setResultsName('subj') +
        uri_part.setResultsName('pred') +
        (Group(uri_part).setResultsName('obj')
         ^ Group(literal_term).setResultsName('objlit')) +
        Literal('>').suppress())
    expr = Forward()
    atom = (triple.setResultsName('triple')
            | Literal('(').suppress() + expr + Literal(')').suppress())
    and_group = Group(atom + ZeroOrMore(and_kw.suppress() + atom))
    or_group = Group(atom + ZeroOrMore(or_kw.suppress() + atom))
    expr << (and_group.setResultsName('and') ^ or_group.setResultsName('or'))
    return expr
Ejemplo n.º 39
0
def _build_parser():
  identifier = Regex(r"[A-Za-z]+(_[A-Za-z]+)*").setParseAction(lambda t: Identifier(t[0]))
  string = QuotedString('"', unquoteResults=True, multiline=True) # FIXME: escaping?
  integer = Regex(r"-?\d+").setParseAction(lambda t: int(t[0]))
  decimal = Regex(r"-?\d+\.\d*").setParseAction(lambda t: float(t[0]))
  value = string | decimal | integer | identifier
  array = (value + OneOrMore(Suppress(",") + value)).setParseAction(tuple)
  levelValue = integer | Regex(r"[X]")

  stanza = Forward()

  block = (identifier + Optional(Suppress("(") + value + Suppress(")"), default=None) + \
          Group(Suppress("{") + ZeroOrMore(stanza) + Suppress("}"))) \
          .setParseAction(lambda t: Block(*t))
  assignment = (identifier + Suppress("=") + (array | value) + Suppress(";")) \
               .setParseAction(lambda t: Assignment(*t))
  levelStatement = (Suppress("LEVEL") + levelValue + Suppress("FOR") + decimal + Suppress(";")) \
                   .setParseAction(lambda t: LevelStatement(*t))
  stanza << (block | assignment | levelStatement)
  return stanza
Ejemplo n.º 40
0
    def initGrammar(self):
        L_Equals = Word("=")
        N_comment = htmlComment()

        N_name = CharsNotIn("{}|[]")
        N_simpleText = SkipTo(
            oneOf(["{{", "|", "[[", "]]", "}}", "'''", "<ref"]))

        N_elements = Forward()
        N_apostrofs = QuotedString("'''").setParseAction(
            lambda s, l, t: {'APOSTROFS': t})
        N_link = nestedExpr(
            opener="[[",
            closer="]]",
            content=N_name +
            Optional("|" + delimitedList(CharsNotIn("[]"), delim="|"))
        ).setParseAction(self.genLink)
        N_header = Group(L_Equals + SkipTo("=") + L_Equals).setParseAction(
            lambda s, l, t: {'HEADER': t})
        N_template = Forward()
        N_key = CharsNotIn("{}|=")
        # N_value = ZeroOrMore(CharsNotIn("{}|")) + ZeroOrMore(N_template + ZeroOrMore(CharsNotIn("{}|"))).setResultsName('VALUE')
        N_keyValues = "|" + delimitedList(
            Group(Optional(N_key) + Optional("=" + N_elements)), delim="|")
        N_label_content = N_template | ("{{" + OneOrMore("!") +
                                        "}}") | CharsNotIn("{}|")
        N_label = nestedExpr(opener="{", closer="}", content=N_label_content)
        N_template << nestedExpr(
            opener="{{", closer="}}", content=N_name +
            Optional(N_keyValues)).setParseAction(self.genTemplate)

        ref_start, ref_end = makeHTMLTags("ref")
        N_named_ref = ref_start + SkipTo(ref_end) + ref_end
        N_named_ref.setParseAction(lambda s, l, t: {'REF': t})

        N_element = N_comment | N_simpleText | N_named_ref | N_apostrofs | N_link | N_header | N_template | N_label

        # N_ref = nestedExpr( opener="<ref>", closer="</ref>", content=N_elements).setParseAction( lambda s,l,t: {'REF' : t} )
        N_elements << ZeroOrMore(N_element)

        self.N_S = N_elements
Ejemplo n.º 41
0



# define grammar
startContext          = "start_context" + Word(alphanums)
variable              = Word(alphanums)
attribute             = Word(alphanums)+ "::"+ Word(alphanums)
freeformText          = "freeform"+"("+ OneOrMore(Word(alphanums))+")"
comparison            = Literal("=") | Literal("<") | Literal(">")
setAttributeValue     = attribute + Literal("=") + freeformText | attribute + comparison + variable 
setVariableFromVar    = variable + Literal("=") + freeformText | variable + comparison + variable 
setVariableFromAttr   = variable + comparison + attribute 
argument              = Word(alphanums) + Optional(",")
funcDef               = Word(alphanums) + "(" + OneOrMore(argument) + ")"
comment               = QuotedString('/*', endQuoteChar='*/')

line                  = startContext + ";" | setAttributeValue + ";"  + setVariableFromAttr + ";" | setVariableFromVar + ";" | funcDef + ";" | comment
# grammar to be exported
grammar               = OneOrMore(line)


def parse_knowledge_tree(s):
    """ return knowledge tree after parsing with the grammar"""
    grammar.parseString(s)
    return kt



#parse actions
setAttributeValue.setParseAction(update_attribute)
Ejemplo n.º 42
0
    def __init__(self):
        intNum = Word(nums)
        floatNum = Combine(intNum + Optional("." + intNum))
        string = QuotedString("'") | QuotedString('"')
        regex = QuotedString("/")
        ident = Word( alphas, alphanums + "_" )
        time_period = Keyword("minutes") | Keyword("seconds")

        ordering = Keyword("unique") | Keyword("random")
        string_type = Keyword("random").setResultsName("ordering") + \
            Keyword("string").setResultsName("data_type") + \
            Keyword("of") + Keyword("length") + \
            intNum.setResultsName("length")
        numeric_type = ordering.setResultsName("ordering") + \
            Keyword("number").setResultsName("data_type") + Keyword("from") + \
            floatNum.setResultsName("min") + Keyword("to") + \
            floatNum.setResultsName("max")
        var_type = string_type | numeric_type
        var = Group(Keyword("var").setResultsName("type") + \
            ident.setResultsName("name") +  Keyword("is") + \
            Keyword("a") + var_type)


        ident_list = delimitedList( ident )
        using_ordering = Keyword("randomly") | Keyword("sequentially")
        using = Group(Keyword("using").setResultsName("type") + \
            ident_list.setResultsName("vars")  + Keyword("from") + \
            string.setResultsName("filename") + \
            using_ordering.setResultsName("ordering"))

        pause = Group(Keyword("pause").setResultsName("type") + \
            Keyword("between") + \
            intNum.setResultsName("lower_time") + Keyword("and") + \
            intNum.setResultsName("upper_time") + Keyword("seconds"))

        get = Keyword("get").setResultsName("method")
        post = Keyword("post").setResultsName("method")
        put = Keyword("put").setResultsName("method")
        delete = Keyword("delete").setResultsName("method")
        method = (get | post | put | delete).setResultsName("type")

        url = string.setResultsName("url")
        data = Keyword("with") + Keyword("data") + \
            string.setResultsName("data")

        match = Group( \
            Keyword("ensure") + Keyword("match") + \
            regex.setResultsName("regex"))
        match_list = Group(OneOrMore(match)).setResultsName("matches")

        request = Group(method + \
            Optional(Keyword("all")).setResultsName("all") + \
            url + Optional(data) + Optional(match_list)).setName("request")
        action = request | pause | var | using
        action_list = \
            Group(OneOrMore(action)).setResultsName("actions")

        session = Group( Keyword("create") + \
            Keyword("session") + Keyword("with") + \
            Keyword("weight") + \
            intNum.setResultsName("weight")  + Keyword("as") + \
            string.setResultsName("name")  + \
            ":" + action_list)
        session_list = OneOrMore(session).setResultsName("sessions")
        
        spawn = Group( Keyword("spawn") + \
            Keyword("users") + Keyword("every") + \
            intNum.setResultsName("user_time") + \
            time_period.setResultsName("user_time_units") + \
            Keyword("for") + \
            intNum.setResultsName("max_duration") + \
            time_period.setResultsName("max_duration_units") + \
            Optional( Keyword("up") + Keyword("to") + \
            intNum.setResultsName("max_users") +  Keyword("users")))
        spawn_list = OneOrMore(spawn).setResultsName("spawns")
        load = Group( Keyword("create") + Keyword("load") + ":" + \
            spawn_list).setResultsName("load")

        comment = "#" + restOfLine

        script = session_list + load
        script.ignore(comment)

        self.grammar = script
Ejemplo n.º 43
0
# from the RFCs ABNF description
nilvalue = Word("-")
digit = Regex("[0-9]{1}")
nonzero_digit = Regex("[1-9]{1}")
printusascii = printables
sp = White(" ", exact=1)
octet = Regex("[\x00-\xFF]")
utf_8_string = Regex("[\x00-\xFF]*")
BOM = "\xef\xbb\xbf"
bom = Regex(BOM)
msg_utf8 = bom + utf_8_string
msg_any = utf_8_string
msg = Combine(Or([msg_utf8, msg_any])).setResultsName("MSG")
sd_name = CharsNotIn('= ]"', 1, 32)
param_name = sd_name.setResultsName("SD_PARAM_NAME")
param_value = QuotedString(quoteChar='"', escChar="\\", multiline=True)
param_value = param_value.setResultsName("SD_PARAM_VALUE")
sd_id = sd_name.setResultsName("SD_ID")
sd_param = Group(param_name + Regex("=") + param_value)
sd_params = Group(ZeroOrMore(Group(sp + sd_param.setResultsName("SD_PARAM"))))
sd_element = Group("[" + sd_id + sd_params.setResultsName("SD_PARAMS") + "]")
sd_element = sd_element.setResultsName("SD_ELEMENT")
sd_elements = Group(OneOrMore(sd_element))
structured_data = Or([nilvalue, sd_elements.setResultsName("SD_ELEMENTS")])
structured_data = structured_data.setResultsName("STRUCTURED_DATA")
time_hour = Regex("0[0-9]|1[0-9]|2[0-3]")
time_minute = Regex("[0-5][0-9]")
time_second = time_minute
time_secfrac = Regex("\.[0-9]{1,6}")
time_numoffset = Or([Regex("\+"), Regex("-")]) + time_hour + ":" + time_minute
time_offset = Or([Regex("Z"), time_numoffset])
Ejemplo n.º 44
0
def _define_grammar():
    """
    Creates and returns a copy of the selector grammar.

    Wrapped in a function to avoid polluting the module namespace.
    """
    expr = Forward()

    label_name = Word(LABEL_CHARS)
    label_name.setParseAction(LabelNode)

    string_literal = QuotedString('"') | QuotedString("'")
    string_literal.setParseAction(LiteralNode)

    set_literal = (Suppress("{") +
                   delimitedList(QuotedString('"') | QuotedString("'"), ",") +
                   Suppress("}"))
    set_literal.setParseAction(SetLiteralNode)

    eq_comparison = label_name + Suppress("==") + string_literal
    eq_comparison.setParseAction(LabelToLiteralEqualityNode)

    not_eq_comparison = label_name + Suppress("!=") + string_literal
    not_eq_comparison.setParseAction(InequalityNode)

    in_comparison = label_name + Suppress(Keyword("in")) + set_literal
    in_comparison.setParseAction(LabelInSetLiteralNode)

    not_in = Suppress(Keyword("not") + Keyword("in"))
    not_in_comparison = label_name + not_in + set_literal
    not_in_comparison.setParseAction(NotInNode)

    has_check = (Suppress("has(") +
                 Word(LABEL_CHARS) +
                 Suppress(")"))
    has_check.setParseAction(HasNode)

    # For completeness, we allow an all() to occur in an expression like
    # "! all()".  Note: we special-case the trivial selectors "" and
    # "all()" below for efficiency.
    all_op = (Suppress("all()"))
    all_op.setParseAction(AllNode)

    comparison = (eq_comparison |
                  not_eq_comparison |
                  in_comparison |
                  not_in_comparison |
                  has_check |
                  all_op)

    paren_expr = (Suppress("(") + expr + Suppress(")"))

    value = ZeroOrMore("!") + (comparison | paren_expr)
    value.setParseAction(simplify_negation_node)

    and_expr = value + ZeroOrMore(Suppress("&&") + value)
    and_expr.setParseAction(simplify_and_node)

    or_expr = and_expr + ZeroOrMore(Suppress("||") + and_expr)
    or_expr.setParseAction(simplify_or_node)

    expr << or_expr

    grammar = expr + StringEnd()
    return grammar
Ejemplo n.º 45
0
VARIABLE.setParseAction(lambda s, l, t: Variable(t[0]))

VERSION_CMP = (
    L("===") |
    L("==") |
    L(">=") |
    L("<=") |
    L("!=") |
    L("~=") |
    L(">") |
    L("<")
)

MARKER_OP = VERSION_CMP | L("not in") | L("in")

MARKER_VALUE = QuotedString("'") | QuotedString('"')
MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0]))

BOOLOP = L("and") | L("or")

MARKER_VAR = VARIABLE | MARKER_VALUE

MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR)
MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0]))

LPAREN = L("(").suppress()
RPAREN = L(")").suppress()

MARKER_EXPR = Forward()
MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN)
MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR)
Ejemplo n.º 46
0
dcsToken = Keyword("DEFAULT") + Keyword("CHARACTER") + Keyword("SET")
useToken = Keyword("USE")
defaultToken = Keyword("DEFAULT")
unsignedToken = Keyword("UNSIGNED")
autoincrementToken = Keyword("AUTO_INCREMENT")
autoincrementToken.setParseAction(lambda toks: ["PRIMARY KEY AUTOINCREMENT"])
keyToken = Keyword("KEY")
primaryToken = Keyword("PRIMARY")
uniqueToken = Keyword("UNIQUE")
insertToken = Keyword("INSERT")
intoToken = Keyword("INTO")
valuesToken = Keyword("VALUES")

ident = Word(alphas, alphanums + "_$" ) ^ QuotedString('"') ^ QuotedString("`")
ident.setParseAction(lambda toks: ['"%s"' % toks[0]])
string = QuotedString("'",multiline=True)
string.setParseAction(lambda toks: ["'%s'" % toks[0]])

columnName = delimitedList( ident, ".",combine=True)
tableName = delimitedList( ident, ".",combine=True)
dataType = Word(alphas) + Combine(Optional(Literal("(") + (Word(nums) ^ delimitedList(string,combine=True)) + Literal(")"))) + ZeroOrMore(nnToken ^ autoincrementToken ^ (defaultToken + (string ^ nullToken)) ^ unsignedToken.suppress() )
dataType.setParseAction(convert_datatypes)

columnDescription = Group(ident + dataType)
keyDescription = Optional(primaryToken ^ uniqueToken) + keyToken + Optional(ident) + Literal("(") + delimitedList(ident + Optional(Literal("(") + Word(nums) + Literal(")"))) + Literal(")")

createTableStmt = Group(createToken + tableToken + ifneToken + ident + Literal("(")) + delimitedList(columnDescription ^ keyDescription.suppress()) + Group(Literal(")")) + Optional(autoincrementToken + Literal("=") + Word(nums)).suppress()
createTableStmt.setParseAction(rebuild_createtable)


createDataBaseStmt = Group(createToken + databaseToken + ident +  dcsToken + Word(alphanums)+ collateToken + ident)
Ejemplo n.º 47
0
    def __init__(self, get_obfuscated):
        """BNF grammar for source statements.

        Parameters
        ----------
        get_obfuscated : function
            Function to return the obfuscated name for an identifier.
        """
        self.get_obfuscated = get_obfuscated

        self.directive = oneOf("#:")
        self.comment = ~self.directive + pythonStyleComment

        self.separator = Word("~!@$%^&*()+`-={}|[]:;<>?,/.", max=2)

        self.string = \
            QuotedString(quoteChar='"', escChar='\\', multiline=False,
                         unquoteResults=False) |\
            QuotedString(quoteChar="'", escChar='\\', multiline=False,
                         unquoteResults=False)

        self.doc_string = \
            QuotedString(quoteChar='"""', escChar='\\', multiline=True,
                         unquoteResults=False) |\
            QuotedString(quoteChar="'''", escChar='\\', multiline=True,
                         unquoteResults=False)
        self.string_or_doc = self.doc_string | self.string
        self.triple_quote = Literal("'''") | Literal('"""')

        self.e = Literal('E') | Literal('e')
        self.point = Literal('.')

        self.plusorminus = Literal('+') | Literal('-')
        self.number = Word(nums)
        self.integer = Combine(Optional(self.plusorminus) + self.number)
        self.fnumber = Combine(
            self.integer +
            Optional(self.point + Optional(self.number)) +
            Optional(self.e + self.integer))

        self.tab = Literal('    ')

        self.ident = Word(alphas+'_', alphanums+'_')
        self.conseq_idents_numbs = OneOrMore(self.ident | self.fnumber)
        self.attrib = self.ident + OneOrMore('.'+self.ident)

        self.statement = (
            ZeroOrMore(
                (self.directive |
                 self.tab |
                 self.conseq_idents_numbs |
                 self.separator |
                 self.string_or_doc |
                 self.triple_quote)
            ) + Optional(self.comment).suppress()
        )

        self.attribs = (
            ZeroOrMore(
                (self.directive.suppress() |
                 self.tab.suppress() |
                 self.attrib |
                 self.ident.suppress() |
                 self.separator.suppress() |
                 self.fnumber.suppress() |
                 self.string_or_doc.suppress() |
                 self.triple_quote.suppress())
            ) + Optional(self.comment).suppress()
        )

        self.conseq_idents = (
            ZeroOrMore(
                (self.directive.suppress() |
                 self.tab.suppress() |
                 self.ident |
                 self.separator.suppress() |
                 self.fnumber.suppress() |
                 self.string.suppress())
            ) + Optional(self.comment).suppress()
        )

        self.conseq_idents_no_obfuscate = (
            ZeroOrMore(
                (self.directive.suppress() |
                 self.tab.suppress() |
                 self.ident |
                 self.separator.suppress() |
                 self.fnumber.suppress() |
                 self.string_or_doc.suppress() |
                 self.triple_quote.suppress())
            ) + Optional(self.comment).suppress()
        )

        self.attribs.setParseAction(self.add_attribs_reserveds)
        self.conseq_idents.setParseAction(self.add_conseq_idents)
        self.conseq_idents_no_obfuscate.setParseAction(
            self.add_conseq_idents_no_obfuscate)
        self.conseq_idents_numbs.setParseAction(
            self.transform_conseq_ident_numbs)
        self.directive.setParseAction(self.transform_directive)
Ejemplo n.º 48
0
    ZeroOrMore(Suppress("-") + _label_part) +
    Suppress("]")
).setParseAction(tokenize_override_ps)

# Looks like: [subject-group(Some text Goes Here)]
subject_group = (
    context_certainty +
    Suppress("[subject-group") +
    QuotedString(quoteChar='(', endQuoteChar=')').setResultsName("subgroup") +
    Suppress("]")
).setParseAction(lambda m: tokens.Context(
    [None, 'Subjgrp:' + subjgrp_label(m.subgroup, [])], bool(m.certain)))

# Phrases like '“Nonimmigrant visa”' become 'p12345678'
_double_quote_label = QuotedString(
    quoteChar=u'“', endQuoteChar=u'”'
).setParseAction(lambda m: "p{}".format(hash_for_paragraph(m[0])))
# Phrases like "definition for the term “Nonimmigrant visa”" become a
# paragraph token with the appropriate paragraph label set
definition = (
    Marker("definition") +
    (Marker("of") | Marker("for")) +
    Optional(Marker("the") + Marker("term")) +
    _double_quote_label.copy().setResultsName("paragraph")
).setParseAction(lambda m: tokens.Paragraph(paragraphs=[m.paragraph]))

#   grammar which captures all of these possibilities
token_patterns = QuickSearchable(
    put_active | put_passive | post_active | post_passive |
    delete_active | delete_passive | move_active | move_passive |
    designate_active | reserve_active |
Ejemplo n.º 49
0
from pyparsing import Literal, OneOrMore, QuotedString, SkipTo, stringEnd


"""Givent a string, this parser skips to the starting charcters '<%',
returning everything it skips over as a token. Everything that is
between '<%=' and '%>' are returned as a token labeled
assignment. This is repeated until we run out of '<%' at which point
we return everything up to the end of a string as a token."""

startExp = Literal('<%')
assignExp = QuotedString('<%=', endQuoteChar='%>', multiline=True)
assignExp = assignExp.setResultsName('assignment')

ERBParser = OneOrMore(SkipTo(startExp) + assignExp) + SkipTo(stringEnd)
ERBParser.leaveWhitespace()

class ERBTemplate(object):
    def loadFromFile(self, fileName):
        tokens = ERBParser.parseFile(fileName)

        # Create a dictionary where the key is the position the token
        # was found and and the value is the token's name as set by setResultsName

        typeByPos  = dict( [ (v[1],k) for (k,vlist) in tokens._ParseResults__tokdict.items()
                             for v in vlist ] )
        self.typeByPos = typeByPos
        self.tokens = tokens

    def render(self, context):
        tokens = self.tokens
        tokenCount = len(tokens)
Ejemplo n.º 50
0
import re
import jinja2
import pyparsing
import bleach
from .attachments import THUMB_PATTERN
from pyparsing import QuotedString, ParserElement, LineStart, LineEnd, SkipTo, OneOrMore, restOfLine
from .util import mime2thumb_ext

ParserElement.setDefaultWhitespaceChars(' \t')

EOL = LineEnd()
SOL = LineStart()

strong = QuotedString("**") | QuotedString(quoteChar="[b]", endQuoteChar="[/b]")
strong.setParseAction(lambda x: "<strong>%s</strong>" % x[0])

italic = QuotedString("*", escChar='\\') | QuotedString(quoteChar="[i]", endQuoteChar="[/i]")
italic.setParseAction(lambda x: "<i>%s</i>" % x[0])

underline = QuotedString("__") | QuotedString(quoteChar="[u]", endQuoteChar="[/u]")
underline.setParseAction(lambda x: "<u>%s</u>" % x[0])

strike = QuotedString(quoteChar="[s]", endQuoteChar="[/s]")
strike.setParseAction(lambda x: "<s>%s</s>" % x[0])

sup = QuotedString(quoteChar="[sup]", endQuoteChar="[/sup]")
sup.setParseAction(lambda x: "<sup>%s</sup>" % x[0])

sub = QuotedString(quoteChar="[sub]", endQuoteChar="[/sub]")
sub.setParseAction(lambda x: "<sub>%s</sub>" % x[0])
Ejemplo n.º 51
0
class ObfuscateBNF(object):
    __metaclass__ = abc.ABCMeta

    def __init__(self, get_obfuscated):
        """BNF grammar for source statements.

        Parameters
        ----------
        get_obfuscated : function
            Function to return the obfuscated name for an identifier.
        """
        self.get_obfuscated = get_obfuscated

        self.directive = oneOf("#:")
        self.comment = ~self.directive + pythonStyleComment

        self.separator = Word("~!@$%^&*()+`-={}|[]:;<>?,/.", max=2)

        self.string = \
            QuotedString(quoteChar='"', escChar='\\', multiline=False,
                         unquoteResults=False) |\
            QuotedString(quoteChar="'", escChar='\\', multiline=False,
                         unquoteResults=False)

        self.doc_string = \
            QuotedString(quoteChar='"""', escChar='\\', multiline=True,
                         unquoteResults=False) |\
            QuotedString(quoteChar="'''", escChar='\\', multiline=True,
                         unquoteResults=False)
        self.string_or_doc = self.doc_string | self.string
        self.triple_quote = Literal("'''") | Literal('"""')

        self.e = Literal('E') | Literal('e')
        self.point = Literal('.')

        self.plusorminus = Literal('+') | Literal('-')
        self.number = Word(nums)
        self.integer = Combine(Optional(self.plusorminus) + self.number)
        self.fnumber = Combine(
            self.integer +
            Optional(self.point + Optional(self.number)) +
            Optional(self.e + self.integer))

        self.tab = Literal('    ')

        self.ident = Word(alphas+'_', alphanums+'_')
        self.conseq_idents_numbs = OneOrMore(self.ident | self.fnumber)
        self.attrib = self.ident + OneOrMore('.'+self.ident)

        self.statement = (
            ZeroOrMore(
                (self.directive |
                 self.tab |
                 self.conseq_idents_numbs |
                 self.separator |
                 self.string_or_doc |
                 self.triple_quote)
            ) + Optional(self.comment).suppress()
        )

        self.attribs = (
            ZeroOrMore(
                (self.directive.suppress() |
                 self.tab.suppress() |
                 self.attrib |
                 self.ident.suppress() |
                 self.separator.suppress() |
                 self.fnumber.suppress() |
                 self.string_or_doc.suppress() |
                 self.triple_quote.suppress())
            ) + Optional(self.comment).suppress()
        )

        self.conseq_idents = (
            ZeroOrMore(
                (self.directive.suppress() |
                 self.tab.suppress() |
                 self.ident |
                 self.separator.suppress() |
                 self.fnumber.suppress() |
                 self.string.suppress())
            ) + Optional(self.comment).suppress()
        )

        self.conseq_idents_no_obfuscate = (
            ZeroOrMore(
                (self.directive.suppress() |
                 self.tab.suppress() |
                 self.ident |
                 self.separator.suppress() |
                 self.fnumber.suppress() |
                 self.string_or_doc.suppress() |
                 self.triple_quote.suppress())
            ) + Optional(self.comment).suppress()
        )

        self.attribs.setParseAction(self.add_attribs_reserveds)
        self.conseq_idents.setParseAction(self.add_conseq_idents)
        self.conseq_idents_no_obfuscate.setParseAction(
            self.add_conseq_idents_no_obfuscate)
        self.conseq_idents_numbs.setParseAction(
            self.transform_conseq_ident_numbs)
        self.directive.setParseAction(self.transform_directive)

    ###############
    # Parse actions
    ###############
    def add_conseq_idents(self, conseq_idents_list):
        """Add names to obfuscate to identifiers table.

        Parameters
        ----------
        conseq_idents_list : list
        """
        if 'import' not in conseq_idents_list[:] and \
                'except' not in conseq_idents_list[:]:
            add_identifiers(set(conseq_idents_list))

    def add_conseq_idents_no_obfuscate(
            self, conseq_idents_no_obfuscate_list):
        """Add names that are not obfuscated to identifiers table.

        Parameters
        ----------
        conseq_idents_no_obfuscate_list : list
        """
        # If an except error was not added to reserved list, don't obfuscate it
        if 'import' not in conseq_idents_no_obfuscate_list[:] and \
                'except' not in conseq_idents_no_obfuscate_list[:]:
            add_identifiers(set(conseq_idents_no_obfuscate_list),
                            do_obfuscate=False)

    def add_attribs_reserveds(self, attribs_list):
        """Add attributes of reserved names to reserved list.

        Take a list of attributes strings from a source statement, break
        it into lists of objects with their attributes, and add attributes
        that follow a reserved name to the reserved list.

        Example
         ------
         If r is reserved, then
            a.r.c + d.r.e
        would add c and e to reserveds.

        Parameters
        ----------
        attribs_list : list
        """
        if attribs_list:
            # Create an ordered list of attribute parents
            # Ex. a.b.c => [a, b]
            _attrib_list = [attribs_list[0]]
            is_last_token_an_attrib = True
            for token in attribs_list[1:]:
                if is_last_token_an_attrib and token != '.':
                    # End of attrib list reached. Process list.
                    add_attribs_reserveds_list(_attrib_list)
                    # Start new attrib list
                    _attrib_list = [token]
                    is_last_token_an_attrib = True
                elif is_last_token_an_attrib and token == '.':
                    is_last_token_an_attrib = False
                elif not is_last_token_an_attrib and token == '.':
                    continue  # Multiple dots, continue attrib list
                elif not is_last_token_an_attrib and token != '.':
                    _attrib_list.append(token)
                    is_last_token_an_attrib = True
            else:
                # Process last list
                if _attrib_list:
                    add_attribs_reserveds_list(_attrib_list)

    def transform_conseq_ident_numbs(self, conseq_ident_list):
        """Allow for non-name tokens in a statement.

        Names start with an alpha or underscore. Obfuscate these name tokens
        and simply copy unchanged other tokens.

        Parameters
        ----------
        conseq_ident_list : list

        Returns
        -------
        statement : str
        """
        return ' '.join([
            self.get_obfuscated(ident) if
            (ident[0].isalpha() or ident[0] == '_') else ident
            for ident in conseq_ident_list
        ])

    def transform_directive(self, directive_list):
        """Create a directive statement."""
        return ''.join([directive_list[0], ' '])
Ejemplo n.º 52
0
def transform_human(text, variables=None):
    """Transform user input with given context.

    Args:
        text (str): User input.
        variables (dict): Variables for purposes of substitution.

    Returns:
        A 2-tuple of: (A human-readable script that Script can parse,
            A list of contextual information for tooltips, etc.)
    """
    if variables is None:
        variables = {} # No mutable default value.
    # these are parseActions for pyparsing.
    def str_literal_to_hex(s, loc, toks):
        for i, t in enumerate(toks):
            toks[i] = ''.join(['0x', t.encode('hex')])
        return toks
    def var_name_to_value(s, loc, toks):
        for i, t in enumerate(toks):
            val = variables.get(t[1:])
            if val:
                toks[i] = val
        return toks
    def implicit_opcode_to_explicit(s, loc, toks):
        """Add "OP_" prefix to an opcode."""
        for i, t in enumerate(toks):
            toks[i] = '_'.join(['OP', t])
        return toks
    def hex_to_formatted_hex(s, loc, toks):
        """Add "0x" prefix and ensure even length."""
        for i, t in enumerate(toks):
            new_tok = t
            # Add '0x' prefix
            if not t.startswith('0x'):
                if t.startswith('x'):
                    new_tok = ''.join(['0', t])
                else:
                    new_tok = ''.join(['0x', t])
            # Even-length string
            if len(new_tok) % 2 != 0:
                new_tok = ''.join([new_tok[0:2], '0', new_tok[2:]])
            toks[i] = new_tok
        return toks
    # ^ parseActions for pyparsing end here.
    str_literal = QuotedString('"')
    str_literal.setParseAction(str_literal_to_hex)
    var_name = Combine(Word('$') + Word(pyparsing.alphas))
    var_name.setParseAction(var_name_to_value)

    # Here we populate the list of contextual tips.

    # Explicit opcode names
    op_names = [str(i) for i in OPCODE_NAMES.keys()]
    op_names_explicit = ' '.join(op_names)
    def is_small_int(op):
        """True if op is one of OP_1, OP_2, ...OP_16"""
        try:
            i = int(op[3:])
            return True
        except ValueError:
            return False
    op_names_implicit = ' '.join([i[3:] for i in op_names if not is_small_int(i)])

    # Hex, implicit (e.g. 'a') and explicit (e.g. '0x0a')
    explicit_hex = Combine(Word('0x') + Word(pyparsing.hexnums) + pyparsing.WordEnd())
    implicit_hex = Combine(pyparsing.WordStart() + OneOrMore(Word(pyparsing.hexnums)) + pyparsing.WordEnd())
    explicit_hex.setParseAction(hex_to_formatted_hex)
    implicit_hex.setParseAction(hex_to_formatted_hex)

    # Opcodes, implicit (e.g. 'ADD') and explicit (e.g. 'OP_ADD')
    explicit_op = pyparsing.oneOf(op_names_explicit)
    implicit_op = Combine(pyparsing.WordStart() + pyparsing.oneOf(op_names_implicit))
    implicit_op.setParseAction(implicit_opcode_to_explicit)

    contexts = pyparsing.Optional(var_name('Variable') |
                                  str_literal('String literal') |
                                  explicit_op('Opcode') |
                                  implicit_op('Opcode') |
                                  explicit_hex('Hex') |
                                  implicit_hex('Hex'))
    matches = [(i[0].asDict(), i[1], i[2]) for i in contexts.scanString(text)]
    context_tips = []
    for i in matches:
        d = i[0]
        if len(d.items()) == 0: continue
        match_type, value = d.items()[0]
        start = i[1]
        end = i[2]
        context_tips.append( (start, end, value, match_type) )

    # Now we do the actual transformation.

    s = text
    s = var_name.transformString(s)
    s = str_literal.transformString(s)
    s = implicit_op.transformString(s)
    s = implicit_hex.transformString(s)
    s = explicit_hex.transformString(s)
    return s, context_tips
Ejemplo n.º 53
0
        return val


integer = Word(nums)
real = Combine(Word(nums) + '.' + Word(nums))

constant = (Literal('True') |
            Literal('False') |
            Literal('None') |
            Literal('yes') |
            Literal('no')) + WordEnd()

model_reference = Regex(r'([\w\.]*#[\w\.]+)')
variable = Regex(r'([a-zA-Z0-9\._]+)')

string = QuotedString('"', escChar="\\") | QuotedString('\'', escChar="\\")
operand = model_reference | real | integer | constant | string | variable

plusop = oneOf('+ -')
multop = oneOf('* / // %')
groupop = Literal(',')

expr = Forward()

modifier = Combine(Word(alphas + nums) + ':')

integer.setParseAction(EvalInteger)
real.setParseAction(EvalReal)
string.setParseAction(EvalString)
constant.setParseAction(EvalConstant)
variable.setParseAction(EvalVariable)
Ejemplo n.º 54
0
def parse(string=None, filename=None, token=None, lang=None):
    """
    Parse a token stream from or raise a SyntaxError

    This function includes the parser grammar.
    """

    if not lang:
        lang = guess_language(string, filename)

    #
    # End of Line
    #
    EOL = Suppress(lineEnd)
    UTFWORD = Word(unicodePrintables)

    #
    # @tag
    #
    TAG = Suppress('@') + UTFWORD

    #
    # A table
    #
    # A table is made up of rows of cells, e.g.
    #
    #   | column 1 | column 2 |
    #
    # Table cells need to be able to handle escaped tokens such as \| and \n
    #
    def handle_esc_char(tokens):
        token = tokens[0]

        if token == r'\|':
            return u'|'
        elif token == r'\n':
            return u'\n'
        elif token == r'\\':
            return u'\\'

        raise NotImplementedError(u"Unknown token: %s" % token)

    ESC_CHAR = Word(initChars=r'\\', bodyChars=unicodePrintables, exact=2)
    ESC_CHAR.setParseAction(handle_esc_char)

    #
    # A cell can contain anything except a cell marker, new line or the
    # beginning of a cell marker, we then handle escape characters separately
    # and recombine the cell afterwards
    #
    CELL = OneOrMore(CharsNotIn('|\n\\') + Optional(ESC_CHAR))
    CELL.setParseAction(lambda tokens: u''.join(tokens))

    TABLE_ROW = Suppress('|') + OneOrMore(CELL + Suppress('|')) + EOL
    TABLE_ROW.setParseAction(lambda tokens: [v.strip() for v in tokens])
    TABLE = Group(OneOrMore(Group(TABLE_ROW)))

    #
    # Multiline string
    #
    def clean_multiline_string(s, loc, tokens):
        """
        Clean a multiline string

        The indent level of a multiline string is the indent level of the
        triple-". We have to derive this by walking backwards from the
        location of the quoted string token to the newline before it.

        We also want to remove the leading and trailing newline if they exist.

        FIXME: assumes UNIX newlines
        """

        def remove_indent(multiline, indent):
            """
            Generate the lines removing the indent
            """

            for line in multiline.splitlines():
                if line and not line[:indent].isspace():
                    warn("%s: %s: under-indented multiline string "
                         "truncated: '%s'" %
                         (lineno(loc, s), col(loc, s), line),
                         LettuceSyntaxWarning)

                # for those who are surprised by this, slicing a string
                # shorter than indent will yield empty string, not IndexError
                yield line[indent:]

        # determine the indentation offset
        indent = loc - s.rfind('\n', 0, loc) - 1

        multiline = '\n'.join(remove_indent(tokens[0], indent))

        # remove leading and trailing newlines
        if multiline[0] == '\n':
            multiline = multiline[1:]

        if multiline[-1] == '\n':
            multiline = multiline[:-1]

        return multiline

    MULTILINE = QuotedString('"""', multiline=True)
    MULTILINE.setParseAction(clean_multiline_string)

    # A Step
    #
    # Steps begin with a keyword such as Given, When, Then or And They can
    # contain an optional inline comment, although it's possible to encapsulate
    # it in a string. Finally they can contain a table or a multiline 'Python'
    # string.
    #
    # <variables> are not parsed as part of the grammar as it's not easy to
    # distinguish between a variable and XML. Instead scenarios will replace
    # instances in the steps based on the outline keys.
    #
    STATEMENT_SENTENCE = Group(
        lang.STATEMENT +  # Given, When, Then, And
        OneOrMore(UTFWORD.setWhitespaceChars(' \t') |
                  quotedString.setWhitespaceChars(' \t')) +
        EOL
    )

    STATEMENT = Group(
        STATEMENT_SENTENCE('sentence') +
        Optional(TABLE('table') | MULTILINE('multiline'))
    )
    STATEMENT.setParseAction(Step)

    STATEMENTS = Group(ZeroOrMore(STATEMENT))

    #
    # Background:
    #
    BACKGROUND_DEFN = \
        lang.BACKGROUND('keyword') + Suppress(':') + EOL
    BACKGROUND_DEFN.setParseAction(Background)

    BACKGROUND = Group(
        BACKGROUND_DEFN('node') +
        STATEMENTS('statements')
    )
    BACKGROUND.setParseAction(Background.add_statements)

    #
    # Scenario: description
    #
    SCENARIO_DEFN = Group(
        Group(ZeroOrMore(TAG))('tags') +
        lang.SCENARIO('keyword') + Suppress(':') +
        restOfLine('name') +
        EOL
    )
    SCENARIO_DEFN.setParseAction(Scenario)

    SCENARIO = Group(
        SCENARIO_DEFN('node') +
        STATEMENTS('statements') +
        Group(ZeroOrMore(
            Suppress(lang.EXAMPLES + ':') + EOL + TABLE
        ))('outlines')
    )
    SCENARIO.setParseAction(Scenario.add_statements)

    #
    # Feature: description
    #
    FEATURE_DEFN = Group(
        Group(ZeroOrMore(TAG))('tags') +
        lang.FEATURE('keyword') + Suppress(':') +
        restOfLine('name') +
        EOL
    )
    FEATURE_DEFN.setParseAction(Feature)

    #
    # A description composed of zero or more lines, before the
    # Background/Scenario block
    #
    DESCRIPTION_LINE = Group(
        ~BACKGROUND_DEFN + ~SCENARIO_DEFN +
        OneOrMore(UTFWORD).setWhitespaceChars(' \t') +
        EOL
    )
    DESCRIPTION = Group(ZeroOrMore(DESCRIPTION_LINE | EOL))
    DESCRIPTION.setParseAction(Description)

    #
    # Complete feature file definition
    #
    FEATURE = Group(
        FEATURE_DEFN('node') +
        DESCRIPTION('description') +
        Optional(BACKGROUND('background')) +
        Group(OneOrMore(SCENARIO))('scenarios') +
        stringEnd)
    FEATURE.ignore(pythonStyleComment)
    FEATURE.setParseAction(Feature.add_blocks)

    #
    # Try parsing the string
    #

    if not token:
        token = FEATURE
    else:
        token = locals()[token]

    try:
        if string:
            tokens = token.parseString(string)
        elif filename:
            with open(filename, 'r', 'utf-8') as fp:
                tokens = token.parseFile(fp)
        else:
            raise RuntimeError("Must pass string or filename")

        return tokens
    except ParseException as e:
        if e.parserElement == stringEnd:
            msg = "Expected EOF (max one feature per file)"
        else:
            msg = e.msg

        raise LettuceSyntaxError(
            filename,
            u"{lineno}:{col} Syntax Error: {msg}\n{line}\n{space}^".format(
                msg=msg,
                lineno=e.lineno,
                col=e.col,
                line=e.line,
                space=' ' * (e.col - 1)))
    except LettuceSyntaxError as e:
        # reraise the exception with the filename
        raise LettuceSyntaxError(filename, e.string)
Ejemplo n.º 55
0
# define the parser
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))

constant = oneOf('True False None yes no') + WordEnd(word_characters)

variable = Regex(r'([a-zA-Z0-9\._]+)')
explicit_variable = '$' + Regex(r'([a-zA-Z0-9\._]+)')
current_scope = Literal('$$')

string = (QuotedString("'''", escChar=None, unquoteResults=True) |
          QuotedString('"""', escChar=None, unquoteResults=True) |
          QuotedString('"', escChar="\\", unquoteResults=True) |
          QuotedString('\'', escChar="\\", unquoteResults=True))
regexp = QuotedString('/', escChar=None)
timespan = Combine(Word(nums) + oneOf('ms s m h d'))

current_scope_operand = current_scope
variable_operand = variable
explicit_variable_operand = explicit_variable
integer_operand = integer
real_operand = real
number_operand = real | integer
string_operand = string

groupop = Literal(',')
signop = oneOf('+ -')
multop = oneOf('* / // %')
filterop = oneOf('|')
plusop = oneOf('+ -')