Exemplo n.º 1
0
    def _parse_cut_ami(text):
        """Parse "cut ami" command using pyparsing"""

        # Word == single token
        edctoken = Word(alphanums + '_')
        withtoken = Word(printables.replace('=', ''))

        preamble = Suppress(Literal('cut') + 'ami')

        # e.g. prod-edx-exdapp. Combining into 1 token enforces lack of whitespace
        e_d_c = Combine(
            edctoken('environment') + '-' + edctoken('deployment') + '-' +
            edctoken('cluster'))

        # e.g. cut ami for prod-edx-edxapp. Subsequent string literals are converted when added to a pyparsing object.
        for_from = Suppress('for') + e_d_c('for_edc') + Suppress(
            'from') + e_d_c('from_edc')

        # e.g. with foo=bar bing=baz.
        # Group puts the k=v pairs in sublists instead of flattening them to the top-level token list.
        with_stmt = Suppress('with')
        with_stmt += OneOrMore(
            Group(withtoken('key') + Suppress('=') +
                  withtoken('value')))('overrides')

        # e.g. using ami-deadbeef
        using_stmt = Suppress('using') + Regex('ami-[0-9a-f]{8}')('ami_id')

        # 0-1 with and using clauses in any order (see Each())
        modifiers = Optional(with_stmt('with_stmt')) & Optional(
            using_stmt('using_stmt'))

        # 0-1 verbose and noop options in any order (as above)
        options = Optional(Literal('verbose')('verbose')) & Optional(
            Literal('noop')('noop'))

        pattern = StringStart(
        ) + preamble + options + for_from + modifiers + StringEnd()

        parsed = pattern.parseString(text)
        return {
            'dest_env': parsed.for_edc.environment,
            'dest_dep': parsed.for_edc.deployment,
            'dest_play': parsed.for_edc.cluster,
            'source_env': parsed.from_edc.environment,
            'source_dep': parsed.from_edc.deployment,
            'source_play': parsed.from_edc.cluster,
            'base_ami':
            parsed.using_stmt.ami_id if parsed.using_stmt else None,
            'version_overrides':
            {i.key: i.value
             for i in parsed.with_stmt.overrides}
            if parsed.with_stmt else None,
            'verbose': bool(parsed.verbose),
            'noop': bool(parsed.noop),
        }
Exemplo n.º 2
0
def parse(s):
    path = Word(printables) ^ QuotedString('"')
    option_name = Literal("exec") ^ Literal("include") ^ Literal("exclude")
    option_value = path
    option = Group(option_name + option_value)
    options_list = Group(ZeroOrMore(option))
    path_entry = Group(path + Literal("{") + \
                       options_list + \
                       Literal("}"))
    path_entries_list = ZeroOrMore(path_entry)
    config = StringStart() + path_entries_list + StringEnd()

    entries = []
    for e in config.parseString(remove_comments(s)):
        opts = []
        if len(e) == 4:
            for o in e[2]:
               opts.append(Option(o[0], sanitize(o[1])))
        entries.append(Entry(sanitize(e[0]), opts))

    return entries
Exemplo n.º 3
0
    def parse_query_string(self, query_string):
        # pylint: disable=too-many-locals
        """
        Function that parse the querystring, extracting infos for limit, offset,
        ordering, filters, attribute and extra projections.
        :param query_string (as obtained from request.query_string)
        :return: parsed values for the querykeys
        """

        from pyparsing import Word, alphas, nums, alphanums, printables, \
            ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
            QuotedString, Combine, \
            StringStart as SS, StringEnd as SE, \
            WordEnd as WE, \
            ParseException

        from pyparsing import pyparsing_common as ppc
        from dateutil import parser as dtparser
        from psycopg2.tz import FixedOffsetTimezone

        ## Define grammar
        # key types
        key = Word(f'{alphas}_', f'{alphanums}_')
        # operators
        operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
                    | Literal('=notin=') | Literal('=') | Literal('!=')
                    | Literal('>=') | Literal('>') | Literal('<=')
                    | Literal('<'))
        # Value types
        value_num = ppc.number
        value_bool = (
            Literal('true')
            | Literal('false')).addParseAction(lambda toks: bool(toks[0]))
        value_string = QuotedString('"', escQuote='""')
        value_orderby = Combine(Optional(Word('+-', exact=1)) + key)

        ## DateTimeShift value. First, compose the atomic values and then
        # combine
        #  them and convert them to datetime objects
        # Date
        value_date = Combine(
            Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
            Literal('-') + Word(nums, exact=2))
        # Time
        value_time = Combine(
            Literal('T') + Word(nums, exact=2) +
            Optional(Literal(':') + Word(nums, exact=2)) +
            Optional(Literal(':') + Word(nums, exact=2)))
        # Shift
        value_shift = Combine(
            Word('+-', exact=1) + Word(nums, exact=2) +
            Optional(Literal(':') + Word(nums, exact=2)))
        # Combine atomic values
        value_datetime = Combine(
            value_date + Optional(value_time) + Optional(value_shift) +
            WE(printables.replace('&', ''))
            # To us the
            # word must end with '&' or end of the string
            # Adding  WordEnd  only here is very important. This makes atomic
            # values for date, time and shift not really
            # usable alone individually.
        )

        ########################################################################

        def validate_time(toks):
            """
            Function to convert datetime string into datetime object. The format is
            compliant with ParseAction requirements

            :param toks: datetime string passed in tokens
            :return: datetime object
            """

            datetime_string = toks[0]

            # Check the precision
            precision = len(datetime_string.replace('T', ':').split(':'))

            # Parse
            try:
                dtobj = dtparser.parse(datetime_string)
            except ValueError:
                raise RestInputValidationError(
                    'time value has wrong format. The '
                    'right format is '
                    '<date>T<time><offset>, '
                    'where <date> is expressed as '
                    '[YYYY]-[MM]-[DD], '
                    '<time> is expressed as [HH]:[MM]:['
                    'SS], '
                    '<offset> is expressed as +/-[HH]:['
                    'MM] '
                    'given with '
                    'respect to UTC')
            if dtobj.tzinfo is not None and dtobj.utcoffset() is not None:
                tzoffset_minutes = int(dtobj.utcoffset().total_seconds() // 60)
                return DatetimePrecision(
                    dtobj.replace(tzinfo=FixedOffsetTimezone(
                        offset=tzoffset_minutes, name=None)), precision)

            return DatetimePrecision(
                dtobj.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
                precision)

        ########################################################################

        # Convert datetime value to datetime object
        value_datetime.setParseAction(validate_time)

        # More General types
        value = (value_string | value_bool | value_datetime | value_num
                 | value_orderby)
        # List of values (I do not check the homogeneity of the types of values,
        # query builder will do it somehow)
        value_list = Group(value + OneOrMore(Suppress(',') + value) +
                           Optional(Suppress(',')))

        # Fields
        single_field = Group(key + operator + value)
        list_field = Group(key + (Literal('=in=') | Literal('=notin=')) +
                           value_list)
        orderby_field = Group(key + Literal('=') + value_list)
        field = (list_field | orderby_field | single_field)

        # Fields separator
        separator = Suppress(Literal('&'))

        # General query string
        general_grammar = SS() + Optional(field) + ZeroOrMore(
            separator + field) + \
                          Optional(separator) + SE()

        ## Parse the query string
        try:
            fields = general_grammar.parseString(query_string)

            # JQuery adds _=timestamp a parameter to not use cached data/response.
            # To handle query, remove this "_" parameter from the query string
            # For more details check issue #789
            # (https://github.com/aiidateam/aiida-core/issues/789) in aiida-core
            field_list = [
                entry for entry in fields.asList() if entry[0] != '_'
            ]

        except ParseException as err:
            raise RestInputValidationError(
                'The query string format is invalid. '
                "Parser returned this massage: \"{"
                "}.\" Please notice that the column "
                'number '
                'is counted from '
                'the first character of the query '
                'string.'.format(err))

        ## return the translator instructions elaborated from the field_list
        return self.build_translator_parameters(field_list)
Exemplo n.º 4
0
def parse_query_string(query_string):
    """
    Function that parse the querystring, extracting infos for limit, offset,
    ordering, filters, attribute and extra projections.
    :param query_string (as obtained from request.query_string)
    :return: parsed values for the querykeys
    """

    from pyparsing import Word, alphas, nums, alphanums, printables, \
        ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
        QuotedString, Combine, \
        StringStart as SS, StringEnd as SE, \
        WordStart as WS, WordEnd as WE, \
        ParseException

    from pyparsing import pyparsing_common as ppc
    from dateutil import parser as dtparser
    from psycopg2.tz import FixedOffsetTimezone

    ## Define grammar
    # key types
    key = Word(alphas + '_', alphanums + '_')
    # operators
    operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
                | Literal('=notin=') | Literal('=') | Literal('!=')
                | Literal('>=') | Literal('>') | Literal('<=') | Literal('<'))
    # Value types
    valueNum = ppc.number
    valueBool = (Literal('true')
                 | Literal('false')).addParseAction(lambda toks: bool(toks[0]))
    valueString = QuotedString('"', escQuote='""')
    valueOrderby = Combine(Optional(Word('+-', exact=1)) + key)

    ## DateTimeShift value. First, compose the atomic values and then combine
    #  them and convert them to datetime objects
    # Date
    valueDate = Combine(
        Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
        Literal('-') + Word(nums, exact=2))
    # Time
    valueTime = Combine(
        Literal('T') + Word(nums, exact=2) +
        Optional(Literal(':') + Word(nums, exact=2)) +
        Optional(Literal(':') + Word(nums, exact=2)))
    # Shift
    valueShift = Combine(
        Word('+-', exact=1) + Word(nums, exact=2) +
        Optional(Literal(':') + Word(nums, exact=2)))
    # Combine atomic values
    valueDateTime = Combine(
        valueDate + Optional(valueTime) + Optional(valueShift) +
        WE(printables.translate(None, '&'))  # To us the
        # word must end with '&' or end of the string
        # Adding  WordEnd  only here is very important. This makes atomic
        # values for date, time and shift not really
        # usable alone individually.
    )

    ############################################################################
    # Function to convert datetime string into datetime object. The format is
    # compliant with ParseAction requirements
    def validate_time(s, loc, toks):

        datetime_string = toks[0]

        # Check the precision
        precision = len(datetime_string.replace('T', ':').split(':'))

        # Parse
        try:
            dt = dtparser.parse(datetime_string)
        except ValueError:
            raise RestInputValidationError("time value has wrong format. The "
                                           "right format is "
                                           "<date>T<time><offset>, "
                                           "where <date> is expressed as "
                                           "[YYYY]-[MM]-[DD], "
                                           "<time> is expressed as [HH]:[MM]:["
                                           "SS], "
                                           "<offset> is expressed as +/-[HH]:["
                                           "MM] "
                                           "given with "
                                           "respect to UTC")
        if dt.tzinfo is not None:
            tzoffset_minutes = int(
                dt.tzinfo.utcoffset(None).total_seconds() / 60)

            return datetime_precision(
                dt.replace(tzinfo=FixedOffsetTimezone(offset=tzoffset_minutes,
                                                      name=None)), precision)
        else:
            return datetime_precision(
                dt.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
                precision)

    ########################################################################

    # Convert datetime value to datetime object
    valueDateTime.setParseAction(validate_time)

    # More General types
    value = (valueString | valueBool | valueDateTime | valueNum | valueOrderby)
    # List of values (I do not check the homogeneity of the types of values,
    # query builder will do it in a sense)
    valueList = Group(value + OneOrMore(Suppress(',') + value) +
                      Optional(Suppress(',')))

    # Fields
    singleField = Group(key + operator + value)
    listField = Group(key + (Literal('=in=') | Literal('=notin=')) + valueList)
    orderbyField = Group(key + Literal('=') + valueList)
    Field = (listField | orderbyField | singleField)

    # Fields separator
    separator = Suppress(Literal('&'))

    # General query string
    generalGrammar = SS() + Optional(Field) + ZeroOrMore(separator + Field) + \
                     Optional(separator) + SE()

    ## Parse the query string
    try:
        fields = generalGrammar.parseString(query_string)
        field_dict = fields.asDict()
        field_list = fields.asList()
    except ParseException as e:
        raise RestInputValidationError("The query string format is invalid. "
                                       "Parser returned this massage: \"{"
                                       "}.\" Please notice that the column "
                                       "number "
                                       "is counted from "
                                       "the first character of the query "
                                       "string.".format(e))

    ## return the translator instructions elaborated from the field_list
    return build_translator_parameters(field_list)
Exemplo n.º 5
0
            Sentence).setParseAction(set_head)
Title = (Literal("*") + Sentence).setParseAction(set_title)

Line = StringStart() + (Subhead | Headline | Title)

# Read a todo list, generate a schedule for the scheduler to optimize.
curr_headline = ""

items = []

for line in open(sys.argv[1], "r"):
    s.clear()
    s["label"] = []

    try:
        L = Line.parseString(line)
    except ParseException, err:
        continue

    labels = [label.lower() for label in s["label"]]
    work = s["content"].replace(" ", "-")

    if s["type"] == type_title:
        print "_End_Time_: " + "23:59/" + s["content"]
        continue

    if s["type"] == type_head:
        current_headline = s["content"].strip()
        #		print "Head = ", current_headline
        continue
Exemplo n.º 6
0
Headline = (Word(nums) + Literal(".") + ZeroOrMore(Label) + Sentence).setParseAction(set_head)
Title = (Literal("*") + Sentence).setParseAction(set_title)

Line = StringStart() + (Subhead | Headline | Title);

# Read a todo list, generate a schedule for the scheduler to optimize.
curr_headline = "";

items = [];

for line in open(sys.argv[1], "r"):
	s.clear();
	s["label"] = [];

	try: 
		L=Line.parseString(line);
	except ParseException,err:
		continue;

	labels = [label.lower() for label in s["label"]];
	work = s["content"].replace(" ", "-");

	if s["type"] == type_title:
		print "_End_Time_: " + "23:59/" + s["content"];		
		continue;

	if s["type"] == type_head:
		current_headline = s["content"].strip();
#		print "Head = ", current_headline
		continue;
Exemplo n.º 7
0
                    input_file, line_num, oldline, newline))
            concat_source.append((input_file, line_num, newline))

#####
# Parse the assembly
#####
logging.info("Parsing source")
asm_addr = 0
assembly = []
labels = {}
label_addrs = {}
for input_file, line_num, line in concat_source:
    logging.debug("{:16.16s} {:3d}: IN:  {}".format(input_file, line_num,
                                                    line))
    try:
        match = grammar.parseString(line, parseAll=True).asDict()
    except ParseException:
        print(f"ERROR on line {line_num}: {line}")
        raise
    logging.debug("{:16.16s} {:3d}: OUT: {}".format(input_file, line_num,
                                                    match))

    # comments will result in an empty dict
    if not match:
        continue

    # label: the next opcode/data/whatever will be associated with this label
    if 'label' in match:
        labels[match['label']] = len(assembly)
        label_addrs[len(assembly)] = match['label']
        logging.debug("{:16.16s} {:3d} Label {} => 0x{:04x}".format(
Exemplo n.º 8
0
class Parser:

    def __init__( self ):

        # define grammar

        # definition of a float number
        point       = Literal('.')
        plusOrMinus = Literal('+') | Literal('-')
        number      = Word(nums)
        integer     = Combine( Optional(plusOrMinus) + number )
        floatNumber = Combine( integer + Optional( point + number ) )

        # operators
        plus  = Literal( "+" )
        minus = Literal( "-" )
        mult  = Literal( "*" )
        div   = Literal( "/" )

        # parentheses are discarded
        lPar  = Literal( "(" ).suppress()
        rPar  = Literal( ")" ).suppress()

        # For precedence handling
        addOp  = plus | minus
        multOp = mult | div
        logOp  = Literal( "log" )

        expr = Forward()
        term = Forward()

        atom = floatNumber | ( lPar + expr + rPar )
        factor = Group(logOp + atom) | atom
        term << ( Group(factor + multOp + term ) | factor )
        expr << ( Group(term + addOp + expr ) | term )

        self.bnf = StringStart() + expr + StringEnd()

        self.ops = {
            "+" : lambda a, b: a + b,
            "-" : lambda a, b: a - b,
            "*" : lambda a, b: a * b,
            "/" : lambda a, b: a / b
        }

        self.funcs = {
            "log" : lambda a: log10(a)
        }


    def evaluate( self, node ):
        if isinstance(node, basestring):
            if node in "+-*/":
                return self.ops[node]
            elif node in ["log"]:
                return self.funcs[node]
            else:
                return float(node)
        else:
            if len(node) == 3:
                operator = self.evaluate(node[1]);
                return operator(self.evaluate(node[0]), self.evaluate(node[2]))
            elif len(node) == 2:
                function = self.evaluate(node[0]);
                return function(self.evaluate(node[1]))
            else:
                return self.evaluate(node[0])


    def parse( self, expr ):
        self.exprStack = []
        try:
            result = self.bnf.parseString( expr )
        except ParseException,err:
            raise Exception, 'Parse Failure: ' + str(err)

        return { "parseResult": result, "evaluation": self.evaluate( result ) }