def __defineParser(self): self.parseField = parsley.makeGrammar(""" neq = <letterOrDigit*>:field ws '!=' ws <letterOrDigit*>:value ->Field(field,value,'!=') eq = <letterOrDigit*>:field ws '==' ws <letterOrDigit*>:value ->Field(field,value,'==') lte = <letterOrDigit*>:field ws '<=' ws <digit*'.'?digit*>:value ->Field(field,value,'<=') gte = <letterOrDigit*>:field ws '>=' ws <digit*'.'?digit*>:value ->Field(field,value,'>=') lt = <letterOrDigit*>:field ws '<' ws <digit*'.'?digit*>:value ->Field(field,value,'<') gt = <letterOrDigit*>:field ws '>' ws <digit*'.'?digit*>:value ->Field(field,value,'>') has = <letterOrDigit*>:field ws 'has' ws <(letterOrDigit|',')*>:value ->Field(field,value,'has') in = <letterOrDigit*>:field ws 'in' ws <(letterOrDigit|',')*>:value ->Field(field,value,'in') hasnot = <letterOrDigit*>:field ws 'not has' ws <(letterOrDigit|',')*>:value ->Field(field,value,'hasnot') notin = <letterOrDigit*>:field ws 'not in' ws <(letterOrDigit|',')*>:value ->Field(field,value,'notin') fieldCondition = ws (neq | eq | lte | lt | gte |gt | has | in | hasnot | notin):evalTuple ws -> evalTuple """, {'Field': Field}) self.parse = parsley.makeGrammar(""" neq = <letterOrDigit* ws '!=' ws letterOrDigit*>:s ->str(s) eq = <letterOrDigit* ws '==' ws letterOrDigit*>:s ->str(s) lte = <letterOrDigit* ws '<=' ws digit*'.'?digit*>:s->str(s) gte = <letterOrDigit* ws '>=' ws digit*'.'?digit*>:s ->str(s) lt = <letterOrDigit* ws '<' ws digit*'.'?digit*>:s->str(s) gt = <letterOrDigit* ws '>' ws digit*'.'?digit*>:s ->str(s) has = <letterOrDigit* ws 'has' ws (letterOrDigit|',')* >:s ->str(s) in = <letterOrDigit* ws 'in' ws (letterOrDigit|',')* >:s ->str(s) hasnot = <letterOrDigit* ws 'not has' ws (letterOrDigit|',')* >:s ->str(s) notin = <letterOrDigit* ws 'not in' ws (letterOrDigit|',')* >:s ->str(s) parens = ws '(' ws expr:e ws ')' ws -> e value = parens | neq | eq | lte | lt | gte |gt | has | in | hasnot | notin ws = ' '* and = 'AND' ws expr3:n -> ('AND', n) or = 'OR' ws expr3:n -> ('OR', n) not = 'NOT' ws value:n -> ('NOT', n) checknot = ws (value|not) andor = ws (and | or) expr = expr3:left andor*:right -> performOperations(left, right) expr3 = ws checknot:right -> getVal(right) """, {"performOperations": self.performOperations, 'getVal': self.getVal})
def __init__(self, code, by_name=False, verbose=False): lines = [line.strip() for line in code.strip().split("\n")] self.head = lines[0].replace(Flinterpreter.WHERE, "").strip() body = lines[1:] self.functions = {function[0]: function[1:] for function in map(self.__extract_function, body)} grammar_def = Flinterpreter.GRAMMAR function_eval = None if by_name: grammar_def += Flinterpreter.ARGS_BY_NAME function_eval = self.__evaluate_function_name else: grammar_def += Flinterpreter.ARGS_BY_VALUE function_eval = self.__evaluate_function_value fnames = " | ".join("'{}'".format(name) for name in self.functions.keys()) if len(fnames) == 0: fnames = "'THERE ARE NO EXTRA FUNCTIONS'" grammar_def += "fname = " + fnames self.grammar = makeGrammar(grammar_def, {}) self.verbose = verbose self.eval_functions = { Operator.Constant: lambda x: x[0], Operator.PrimitiveFunction: self.__evaluate_primitive, Operator.IfThenElse: self.__evaluate_ifelse, Operator.Function: function_eval, Operator.Bool: self.__evaluate_bool, }
def _load_parser(self, grammar: str, protocol: Protocol) -> None: """ Load the parser using the given grammar :param grammar: The parser grammar :param protocol: The protocol :return: None """ self.parser = parsley.makeGrammar( grammar, { 'punctuation': string.punctuation, 'ascii_uppercase': string.ascii_uppercase, 'ascii_lowercase': string.ascii_lowercase, 'itertools': itertools, 'Art': Art, 'ArtField': ArtField, 'Field': Field, 'RelLoc': RelLoc, 'Names': Names, 'protocol': protocol, 'Boolean': Boolean, 'Size': Size, 'ArgumentExpression': ArgumentExpression, 'MethodInvocationExpression': MethodInvocationExpression, 'ConstantExpression': ConstantExpression, 'FieldAccessExpression': FieldAccessExpression, 'ThisExpression': ThisExpression, 'IfElseExpression': IfElseExpression, })
def scss(basedir, input, encoding="utf-8") -> Iterator[Tuple[Path, Path]]: """Scans a SCSS input file and its imports for `@import` statements""" p_basedir = as_pathlib_path(basedir) p_input = as_pathlib_path(input) contents = read(p_basedir / p_input).decode(encoding) parts = contents.split(";") imports = [] def capture(arg): imports.append(arg) grammar = parsley.makeGrammar( """ string = ('"' <(~'"' anything)*>:val '"') -> capture(val) | ("'" <(~"'" anything)*>:val "'") -> capture(val) imports = string:first (ws ',' ws string)* import_rule = ws '@' 'i' 'm' 'p' 'o' 'r' 't' ws imports """, {'capture': capture}) for part in parts: part = part.strip() # any "@import "foo"[, "bar", ...]" if not part.startswith("@import"): continue grammar(part).import_rule() for i in imports: p = Path(i + ".scss") yield p_basedir, p yield from scss(p_basedir, p)
def __init__(self): self.parser = parsley.makeGrammar(gqlGrammar, { 're': re, 'GalaxyQuery': GalaxyQuery, 'GalaxyQueryComparison': GalaxyQueryComparison, 'GalaxyQueryAnd': GalaxyQueryAnd })
def build_protocol(self, input_str: str, name: str = None) -> Protocol: self.protocol.set_protocol_name(name) # load grammar with open("input_parsers/packetlang/packetlang_grammar.txt", "r+") as grammarFile: grammar = grammarFile.read() # create parser parser = parsley.makeGrammar( grammar, { "ascii_letters": string.ascii_letters, "ascii_uppercase": string.ascii_uppercase, "ascii_lowercase": string.ascii_lowercase, "new_bitstring": self.new_bitstring, "new_typedef": self.new_typedef, "new_array": self.new_array, "new_structfield": self.new_structfield, "new_struct": self.new_struct, "new_parameter": self.new_parameter, "new_func": self.new_func, "new_constant": self.new_constant, "new_fieldaccess": self.new_fieldaccess, "new_this": self.new_this, "new_methodinvocation": self.new_methodinvocation, "build_tree": self.build_tree, "set_pdus": self.set_pdus }) # parse input parser(input_str.replace(" ", "").replace("\n", "").replace("\t", "")).protocol() return self.protocol
def parse_file(name): config = ConfigFile() bindings = { 'symbols': symbols, 'config': config, 'Statement': Statement, 'RangeStmt': RangeStmt, 'Pool': Pool, 'Subnet': Subnet, 'Class': Class, 'Subclass': Subclass, 'Group': Group, 'Host': Host, 'ConfigFile': ConfigFile, } with open('cyder/management/commands/lib/dhcpd_compare2/' 'dhcp.parsley') as g: grammar = makeGrammar(g.read(), bindings) with open(name) as f: fStr = '' for line in f: if not line.startswith('group'): line = line[:comment.match(line).start(1)] if not line or line[-1] != '\n': line += '\n' fStr += line g = grammar(fStr) g.configFile() return config
def grammar_factory(env={}): return parsley.makeGrammar( """ dot = '.' number = <digit+dot{0,1}digit*>:val -> float(val) if "." in val else int(val) variable = <letter+>:var -> getattr(env, var) attribute = exactly('$')<anything+>:attr -> int(reduce( lambda obj, attr: getattr(obj, attr, None), [env] + attr.split('.'))) parens = '(' ws expr:e ws ')' -> e value = number | variable | attribute | parens add = '+' ws expr2:n -> ('+', n) sub = '-' ws expr2:n -> ('-', n) mul = '*' ws expr3:n -> ('*', n) div = '/' ws expr3:n -> ('/', n) pow = '^' ws value:n -> ('^', n) addsub = ws (add | sub) muldiv = ws (mul | div) expr = expr2:left addsub*:right -> calculate(left, right) expr2 = expr3:left muldiv*:right -> calculate(left, right) expr3 = value:left pow*:right -> calculate(left, right) """, { 'env': env, 'calculate': calculate })
def parse_memory_file(payload): """Create a grammar that can parse the output of 'free' when called with the -c option. Return the results as a list of dictionaries of dictionaries """ memory_grammar = r""" memory_reports = memory_report* memory_report = headers_line mem_line:m buffers_line:b swap_line:s '\n' -> dict(zip(['memory', 'buffers', 'swap'], [m, b, s])) headers_line = ws 'total' ws 'used' ws 'free' ws 'shared' ws 'buffers' ws 'cached' '\n' mem_line = 'Mem:' ws total:t ws used:u ws free:f ws shared:s ws buffers:b ws cached:c '\n' -> dict([t, u, f, s, b, c]) buffers_line = '-/+ buffers/cache:' ws used:u ws free:f '\n' -> dict([u, f]) swap_line = 'Swap:' ws total:t ws used:u ws free:f '\n' -> dict([t, u, f]) num = <digit+> total = num:n -> ('total', n) used = num:n -> ('used', n) free = num:n -> ('free', n) shared = num:n -> ('shared', n) buffers = num:n -> ('buffers', n) cached = num:n -> ('cached', n) """ memory_parser = parsley.makeGrammar(memory_grammar, {}) memory_reports = memory_parser(payload).memory_reports() return memory_reports
def parse(text, parameters): Utilities.logging.info("Parsing FiniteStateMachine language...") # definition of the grammar according to Parsley # http://parsley.readthedocs.io/en/latest/tutorial.html grammar = r""" # the language is collection of transitions separated by white space main = ( ws transition:fsm ws -> fsm)* # a transition are two words separared by an arrow ->. Return an XML string transition = ws word:b ws '->' ws word:e -> xml('transition',xml('begin',b,self.input.position)+xml('end',e,self.input.position),self.input.position) # a word is a collection of letters word = ( language | letters ) language = letters:n ws '<<' code:l '>>' ws -> miniLanguage(n, l, self.input.position) code = <(~('>>') anything)+> letters = <letter+> """ # create the language object language = parsley.makeGrammar(grammar, {'xml': xml, 'miniLanguage': lambda x,y,z : miniLanguage(x, y, z, parameters) }) # parse the text using the grammar parsed_xml_text = '<FiniteStateMachine>'+''.join(language(text).main())+'</FiniteStateMachine>' # Convert xml string to xml object parsed_xml = etree.fromstring(parsed_xml_text) return parsed_xml, parameters
def comp(inputstr): gr = r""" esc = '\\' anything:x -> x symbol = ( anything:x ?(x not in '|*?+()\\') | esc:x ) -> Node(x) group = '(' expr:x ')' -> x repeat = (group | symbol):x ( '*' -> Node('star', x) | '+' -> Node('plus', x) | '?' -> Node('qmark', x) | -> x ) cat = ( cat:x repeat:y -> Node('cat', x, y) | repeat ) or = ( or:x '|' cat:y -> Node('or', x, y) | cat ) expr = or?:x -> x or Node('') ast = expr:x -> Node('root', Node('cat', x, Node('end'))) """ tree = AST() grammar = parsley.makeGrammar( gr, {'special_chars': r'|*?+()\\', 'Node': tree.node}) tree.root = grammar(inputstr).ast() dfa = tree.dfa_dot('dfa', inputstr) return dfa, tree.dot()
def get_parser(grammar): return parsley.makeGrammar(grammar, { 'punctuation': string.punctuation, 'ascii_uppercase': string.ascii_uppercase, 'ascii_lowercase': string.ascii_lowercase, 'itertools': itertools, 'ExtendedDiagrams': Rfc2Xml, 'rfc': Rfc(compliant=True), 'Date': Date, 'Xref': Xref, 'TocItem': TocItem, 'Section': Section, 'List': List, 'Figure': Figure, 'Artwork': Artwork, 'T': T, 'rfc_title_docname': Rfc2Xml.rfc_title_docname, 'rfc_title_abbrev': Rfc2Xml.rfc_title_abbrev, 'front_names': Rfc2Xml.front_names, 'text_paragraph': Rfc2Xml.text_paragraph, 'sections': Rfc2Xml.sections, 'printr': Rfc2Xml.printr })
def __init__(self, expr, metric_group=None, resolution=0.5): def calculate(start, pairs): result = start for op, value in pairs: if op == '+': result += value elif op == '-': result -= value elif op == '*': result *= value elif op == '/': result /= value return result def repmax(reps): return self.metric_group[reps].weight self.expr = expr self.metric_group = metric_group self.resolution = resolution self.parser = parsley.makeGrammar( WeightExpr.grammar, { "calculate": calculate, 'repmax': repmax, 'lb2kg': lambda n: n * 0.45359237 })
def get_grammar(): gr = """ # Declare some terminals. marker_events_begin = LEAF marker_events_end = LEAF unusual_file_access = LEAF access_sensitive_file = LEAF internet_access = LEAF exfil_execution = access_sensitive_file internet_access # Looking at `arp -a` or sending intranet packet probes. possible_network_scan = LEAF network_scan = LEAF # Looking at other principals on the current system, e.g. `ps`. possible_system_scan = LEAF system_scan = LEAF exfil_channel = http_post_activity | ssh_activity exfil_format = compress_activity? encrypt_activity? """ # Turn abbreviated declarations of foo = LEAF into: foo = 'foo' leaf_re = re.compile(r'^(\w+)(\s*=\s*)LEAF$', flags=re.MULTILINE) return parsley.makeGrammar(leaf_re.sub("\\1\\2'\\1'", gr), {})
def traceparse(jsonData): trace = [] def traceit(*a): trace.append(a) JSONParser = makeGrammar(jsonGrammar, {}, tracefunc=traceit) return JSONParser(jsonData).top(), trace
def init(): global parser parser = StyleParser() parser.parser = parsley.makeGrammar(open(spyral._get_spyral_path() + 'resources/style.parsley', "r").read(), {"string": string, "parser": parser, "leval": literal_eval, "Vec2D": spyral.Vec2D})
def __init__(self, rules, root_rule, repository={}): import parsley Grammar.__init__(self) repo={} for k, v in repository.items(): repo[k]=(v, checker_factory(v))[isinstance(v, Grammar)] self.grammar=parsley.makeGrammar(rules, repo) self.root_rule=root_rule
def parse_spec(spec, *args, **kwargs): interpreter = Interpreter(args, kwargs) bindings = dict( interpreter = interpreter, RefChain = RefChain, RefGraph = RefGraph) parser = parsley.makeGrammar(grammar, bindings) return parser(spec).spec()
def __init__(self, rules, root_rule, repository={}): import parsley Grammar.__init__(self) repo = {} for k, v in repository.items(): repo[k] = (v, checker_factory(v))[isinstance(v, Grammar)] self.grammar = parsley.makeGrammar(rules, repo) self.root_rule = root_rule
def __init__(self, rules, root_rule="expr", repository=None): import parsley Grammar.__init__(self) repo=dict(repository or {}) for key in repo: if isinstance(repo[key], Grammar): repo[key] = checker_factory(repo[key]) self.grammar=parsley.makeGrammar(rules, repo) self.root_rule=root_rule
def __init__(self, depends_string): """Construct a Depends instance. :param depends_string: The string description of the requirements that need to be satisfied. See the bindep README.rst for syntax for the requirements list. """ parser = makeGrammar(grammar, {})(depends_string) self._rules = parser.rules()
def __init__(self, grammar_fn=__default_grammar_fn, expose_all_rules=False): self._grammar_fn = grammar_fn self._grammar = parsley.makeGrammar( open(grammar_fn, "r").read(), { "vvhgvs": vvhgvs, "bioutils": bioutils, "copy": copy }) self._logger = logging.getLogger(__name__) self._expose_rule_functions(expose_all_rules)
def parse(f): g = parsley.makeGrammar( grammar_source, {'string': string, 'Plane': Plane, 'np': np, 'Entity': Entity} ) map_source = f.read() entities = g(map_source).map() return entities
def init(): """ Initializes the Styler. """ global parser parser = StyleParser() style_file = open(spyral._get_spyral_path() + "resources/style.parsley").read() parser.parser = parsley.makeGrammar( style_file, {"string": string, "parser": parser, "leval": literal_eval, "Vec2D": spyral.Vec2D} )
def make_evaluator(env): def setname(name, value): env[name] = value return value return makeGrammar(eval_grammar, {'primitive_funcs': primitive_funcs, 'env': env, 'setname': setname, 'Function': Function})
def parse_dsl(self, lang): where = [] conjunctions = [] errors = "" if len(lang.strip()) == 0: return {'where_string': "", 'values': []} try: def ex(left, op, right): e = [left, op, right] where.append(e) return (left, op, right) def q(op, ex): # data.append(op) return (op, ex) def conj(op, ex): conjunctions.append(op) return (op, ex) x = parsley.makeGrammar(""" string = '"' (~'"' anything)*:s '"' -> ''.join(s) expr = string:left ws ('='|'!='|'>'|'<'):compop ws string:right -> ex(left, compop, right) and = 'and' ws query2:n -> conj('and', n) or = 'or' ws query2:n -> conj('or', n) conj = ws (and | or) query2 = expr:e1 conj*:cnj -> q(cnj, e1) query = query2:e1 conj*:cnj -> q(cnj, e1) """, {"ex" : ex, "q" : q, "conj" : conj }) x(lang).query() i = 0 max = len(conjunctions) where_string = "where " values = [] for ex in where: t1, op, t2 = ex # where_string += "lower(" + t1 + ")" where_string += "lower(" + t1 + ")" where_string += " " + op + "lower( %s )" if t1 == "ip": t2 = int(netaddr.IPAddress(t2)) values.append(t2) if i < max: where_string += conjunctions[i] + " " i += 1 # print where_string # print values return { 'where_string' : where_string, 'values' : values} except Exception, e: errors = str(e) return {'error': errors }
def make_parsley_grammar(cls): """Return a parsley parsing expression grammar.""" return parsley.makeGrammar( """ not_digit = anything:x ?(x not in "0123456789") not_d_or_p = anything:x ?(x not in "0123456789()") ref = not_d_or_p* '(' digit{1, 3} ')' not_d_or_p* year = not_d_or_p* <digit{4}>:y not_d_or_p* -> int(y) date = ref? year:y ref? -> (y, 1, 1) """, {})
def parse(context, value): if not value: raise RequestParseError('Must provide a value for $filter') try: grammar = parsley.makeGrammar( filter_grammar, dict(compose=compose, ops=gops, get_column=colgetter(context['sqlobj']))) except EOFError: raise RequestParseError('Bad $filter {}'.format(value)) context['sqlobj'] = context['sqlobj'].where(grammar(value).expr())
def parse(s, extra=None, bindings=None): if not bindings: bindings = copy.deepcopy(DEFAULT_BINDINGS) if extra: bindings["extra"] = extra global PARSER if not PARSER: PARSER = parsley.makeGrammar(grammar.GRAMMAR, {"lookup": bindings.__getitem__}) return PARSER(s).specification()
def header(self, request, tag): text = """ blah blah *blah* """ BlogGrammar = makeGrammar(blog_grammar, {"tags": tags}) #output = BlogGrammar("*Hello!*").paragraphs() output = BlogGrammar(text).paragraphs() return tag(output)
def _make_initial_grammar(self): g = parsley.makeGrammar( FeeParser.basegrammar, { "match": re.match, "GlyphSelector": GlyphSelector }, unwrap=True, ) g.globals["parser"] = self g.rule_callRule = callRule g.valid_verbs = ["LoadPlugin"] return g
def load_grammar(compile=True): if compile: # re/compile if necessary: if not os.path.exists(COMPILED_GRAMMAR_FILE) or \ os.path.getmtime(GRAMMAR_FILE) > os.path.getmtime(COMPILED_GRAMMAR_FILE): compile_grammar() # load it like this - there doesn't seem to be a better user API available: import kgtk.kypher.grammar_compiled as cgc cgram = cgc.createParserClass(ometa.grammar.OMetaBase, {}) return parsley.wrapGrammar(cgram) else: # rebuild the grammar from scratch: return parsley.makeGrammar(KYPHER_GRAMMAR, {})
def parse(url, context): parserfuncts = { 'callable': functools.partial(called, context), 'item': functools.partial(item, context), 'count': functools.partial(count, context), 'value': functools.partial(value, context), 'links': functools.partial(links, context)} grammar = parsley.makeGrammar(url_grammar, parserfuncts) try: grammar(url).path() except parsley.ParseError: raise RequestParseError('Could not parse url')
def init(): """ Initializes the Styler. """ global parser parser = StyleParser() style_file = open(spyral._get_spyral_path() + 'resources/style.parsley').read() parser.parser = parsley.makeGrammar(style_file, {"string": string, "parser": parser, "leval": literal_eval, "Vec2D": spyral.Vec2D})
def compiled_grammar(self): if self._compiled_grammar is None: self._compiled_grammar = parsley.makeGrammar( self.requirement_grammar, { "lookup": self._environment_bindings().get, "VersionRequirement": VersionRequirement, "UrlRequirement": UrlRequirement, "PathRequirement": PathRequirement, "EnvironmentMarker": EnvironmentMarker, }, ) return self._compiled_grammar
def make_token_grammar(): """ Make the token grammar. """ grammar = parsley.makeGrammar(r""" punctuation = :x ?(x in "+-*/!'#$%&\,.:<=>?@^_`;" '"') -> x float = <'-'{0, 1} digit* '.' digit+>:ds -> float(ds) integer = <'-'{0, 1} digit+>:ds -> int(ds) number = float | integer itemlist = ws item:first (ws item)*:rest ws -> [first] + rest | ws item:only ws -> [only] item = infix_rel_operator | expr:e (ws comment)* -> e | word:w (ws comment)* -> w | itemlist | '[' ws quoted_itemlist:q ws ']' -> list(q) | '[' ws ']' -> [] | '(' itemlist:lst ')' -> tuple(lst) | (ws comment) quoted_itemlist = ws quoted_item:first (ws quoted_item)*:rest ws -> [first] + rest | ws quoted_item:only ws -> [only] quoted_item = infix_rel_operator | number:n (ws comment)* -> n | word:w (ws comment)* -> w | <(~' ' ~'[' ~']' anything)+>:w (ws comment)* -> w | (ws comment) | '[' ws quoted_itemlist:q ws ']' -> list(q) | '[' ws ']' -> [] infix_rel_operator = '=' | '<>' | '>=' | '<=' word = (word_char+):l -> ''.join(l) word_char = (escaped_char:e -> e) | (~';' unescaped_char:u -> u) unescaped_char = (letterOrDigit:c -> c) | (~';' punctuation:p -> p) escaped_char = '\\' (anything:c -> c) comment = <';' rest_of_line>:c -> Comment(c) rest_of_line = <('\\n' | (~'\n' anything))*> parens = '(' ws expr:e ws ')' -> e value = (number:n ->n) | (parens:p -> p) factor = (value:v -> v) | (word:w -> w) add = '+' ws expr2:n -> ('+', n) mul = '*' ws factor:n -> ('*', n) div = '/' ws factor:n -> ('/', n) addonly = ws add muldiv = ws (mul | div) expr = expr2:left addonly*:right -> calculate(left, right) expr2 = factor:left muldiv*:right -> calculate(left, right) """, {"calculate": calculate, "Comment": Comment}) return grammar
def make_parsley_grammar(cls): """Return a parsley parsing expression grammar.""" return parsley.makeGrammar( """ not_digit = anything:x ?(x not in "0123456789") not_d_or_q = anything:x ?(x not in "0123456789IV()") q_letter = ('I' | 'V'):q -> q ref = ws '(' digit{1, 3} ')' ws | '*' quarter = not_d_or_q* <q_letter{1, 3}>:q not_d_or_q* -> q year = not_digit* <digit{4}>:y not_digit* -> int(y) date = year?:y quarter?:q ref? not_d_or_q* ref? ->(y, q_to_m(q), 1) """, {"q_to_m": cls._quarter_num_to_month})
def __init__(self,grammar_fn=__default_grammar_fn): self._grammar_fn = grammar_fn self._grammar = parsley.makeGrammar(open(grammar_fn,'r').read(),{'hgvs': hgvs, 'bioutils': bioutils}) # define function attributes for each grammar rule, prefixed with 'parse_' # e.g., Parser.parse_c_interval('26+2_57-3') -> Interval(...) #TODO: exclude built-in rules self.rules = [ m.replace('rule_','') for m in dir(self._grammar._grammarClass) if m.startswith('rule_') ] for rule_name in self.rules: att_name = 'parse_' + rule_name rule_fxn = self.__make_parse_rule_function(rule_name) self.__setattr__(att_name,rule_fxn)
def generate_parser(grammarFilename): with open(grammarFilename) as grammarFile: return parsley.makeGrammar( grammarFile.read(), { "ascii_uppercase": string.ascii_uppercase, "ascii_lowercase": string.ascii_lowercase, "ascii_letters": string.ascii_letters, "punctuation": string.punctuation, "rfc": rfc, "get_doc_series": get_doc_series, "get_ipr_code": get_ipr_code, "structure_subsections": structure_subsections, "infer_toc": infer_toc, })
def _split_with_peg(self, value, grammar): if grammar in self.grammars: comp_grammar = self.grammars[grammar] else: comp_grammar = parsley.makeGrammar(grammar, {}) self.grammars[grammar] = comp_grammar try: values = comp_grammar(value).values() except: values = [] values = [six.text_type(split_value) for split_value in values] return pd.Series(values)
def __call__(self, logger): if self._compiled_grammar is None: self._compiled_grammar = makeGrammar( self.requirement_grammar, { "VersionRequirement": VersionRequirement, "UrlRequirement": UrlRequirement, "PathRequirement": PathRequirement, "EnvironmentMarker": EnvironmentMarker, "logger": lambda: self._logger.get_target_logger(), }, ) pass self._logger.set_target_logger(logger) yield self._compiled_grammar
def make_parsley_grammar(cls): """Return a parsley parsing expression grammar.""" return parsley.makeGrammar( """ not_digit = anything:x ?(x not in "0123456789 ") q = not_digit* ws digit:q not_digit* ws -> int(q) y = (ws | not_digit) <digit{2}>:y (ws | not_digit) -> y ref = ws '(' digit{1, 3} ')' ws | '*' date = q:q not_digit* y:y ref? ws anything{0, 3} -> (dob_year(y), q_to_m(q), 1) """, { "q_to_m": cls._quarter_num_to_month, "dob_year": cls._dob_year_to_four })
def _split_with_peg(self, value, grammar): if grammar in self.grammars: comp_grammar = self.grammars[grammar] else: comp_grammar = parsley.makeGrammar(grammar, {}) self.grammars[grammar] = comp_grammar try: values = comp_grammar(value).values() except: values = [] values = [unicode(split_value) for split_value in values] return pd.Series(values)
def make_parsley_grammar(cls): """Return a parsley parsing expression grammar.""" return parsley.makeGrammar( """ not_digit = anything:x ?(x not in "0123456789") y = (ws | not_digit) <digit{4}>:y (ws | not_digit) -> y m = (ws | not_digit) <digit{1, 2}>:m (ws | not_digit) -> int(m) y_m = y:y not_digit* m:m anything* -> (y, m, 1) m_y = m:m not_digit* y:y anything* -> (y, m, 1) only_m = m:m anything* -> (None, m, 1) date = y_m | m_y | only_m """, {})
def parse_infix(text): if not 'infix' in _parsers: grammar = """ subexp = '(' (token | subexp)+:ts ')' ws -> Token('SUBEXP', ts) token = fnumber | number | pow | op | unit expr = (subexp | token)+ number = <digit+>:ds ws -> Token('NUM', int(ds)) fnumber = <digit+>:whole '.' <digit+>:dec ws -> Token('NUM', float(whole + '.' + dec)) unit = <(letter|'_')+>:sym ws -> Token('UNIT', sym) op = ('*' | '/'):sym ws -> Token('MUL', sym) pow = '^' <digit+>:exp ws -> Token('POW', int(exp)) """ _parsers['infix'] = parsley.makeGrammar(grammar, {'Token': Token}) tokenized = _parsers['infix'](text).expr() return cobble(tokenized)
def parse_mapping(filename): """Parse mapping `filename`. Return parsing.""" global parsley, MAPPING_PARSER if parsley is None: raise NotImplementedError("Parsley parsing package must be installed.") if MAPPING_PARSER is None: MAPPING_PARSER = parsley.makeGrammar(MAPPING_GRAMMAR, selectors.SELECTORS) log.verbose("Parsing", repr(filename)) filename = rmap.locate_mapping(filename) with log.augment_exception("Parsing error in", repr(filename), exception_class=exceptions.MappingFormatError): with open(filename) as pfile: parser = MAPPING_PARSER(pfile.read()) header, selector, comment = parser.mapping() return Parsing(header, selector, comment)
def make_grammar(): # TODO: tuple with one value # TODO: make sense of the return values x = parsley.makeGrammar( """ clazzChar = letterOrDigit | '.' clazz = <clazzChar+>:x -> Class(x) # a type or a class item = (:x ?(x in 'TUVWXYZ') -> Type(x) | 'None' -> Type('NoneType') | 'unknown' -> Type('unknown') | ('string' | 'bytestring' | 'bytes' | 'unicode'):x -> StringType(x) | clazz:x -> x) commaRule = ',' ws rule:rule ws -> rule orRule = '|' ws rule:rule ws -> rule rule = ('(' ws rule:firstRule ws commaRule*:ruleList ')' -> Tuple([firstRule] + ruleList) | 'list[' ws rule:x ws ']' -> List(x) | 'dict[' ws rule:first ws ',' ws rule:second ws ']' -> Dict(first, second) | (item:one (ws orRule+:two -> Or([one] + two) | ws '<=' ws rule:two -> BoundedType(one, two) | ws '[' ws rule:two ws commaRule*:ruleList ']' -> ParameterizedType(one, [two] + ruleList) | -> one))) expr = rule:arg (ws '->' ws rule:ret -> Function(arg, ret) | -> arg) """, { "Class": Class, "Type": Type, "BoundedType": BoundedType, "Tuple": Tuple, "StringType": StringType, "List": List, "Dict": Dict, "Function": Function, "ParameterizedType": ParameterizedType, "Or": Or, }, ) return x
def test_eof(self): """ EOF should raise a nice error. """ import parsley g = parsley.makeGrammar(""" dig = '1' | '2' | '3' bits = <dig>+ """, {}) input = '123x321' e = self.assertRaises(ParseError, g(input).dig) self.assertEqual(e.formatError(), dedent(""" 123x321 ^ Parse error at line 1, column 1: expected EOF. trail: [] """))
def parse(self, inputstring, document): self.setup_parse(inputstring, document) self.document = document self.current_node = document self.section_handler = _SectionHandler(document) base = os.path.dirname(os.path.abspath(__file__)) filename = os.path.join(base, "markdown.parsley") with open(filename) as pry_file: self.grammar_raw = pry_file.read() self.grammar = parsley.makeGrammar( self.grammar_raw, dict(builder=self), name='Markdown' ) self.grammar(inputstring + '\n').document() self.finish_parse()
def grammar_factory(env={}): return parsley.makeGrammar(""" dot = '.' number = <digit+dot{0,1}digit*>:val -> float(val) if "." in val else int(val) variable = <letter+>:var -> getattr(env, var) attribute = exactly('$')<anything+>:attr -> int(reduce( lambda obj, attr: getattr(obj, attr, None), [env] + attr.split('.'))) parens = '(' ws expr:e ws ')' -> e value = number | variable | attribute | parens add = '+' ws expr2:n -> ('+', n) sub = '-' ws expr2:n -> ('-', n) mul = '*' ws expr3:n -> ('*', n) div = '/' ws expr3:n -> ('/', n) pow = '^' ws value:n -> ('^', n) addsub = ws (add | sub) muldiv = ws (mul | div) expr = expr2:left addsub*:right -> calculate(left, right) expr2 = expr3:left muldiv*:right -> calculate(left, right) expr3 = value:left pow*:right -> calculate(left, right) """, {'env': env, 'calculate': calculate})
def __init__(self, expr, metric_group=None, resolution=0.5): def calculate(start, pairs): result = start for op, value in pairs: if op == '+': result += value elif op == '-': result -= value elif op == '*': result *= value elif op == '/': result /= value return result def repmax(reps): return self.metric_group[reps].weight self.expr = expr self.metric_group = metric_group self.resolution = resolution self.parser = parsley.makeGrammar( WeightExpr.grammar, {"calculate": calculate, 'repmax': repmax, 'lb2kg': lambda n: n * 0.45359237})
_Comment = collections.namedtuple('Comment', ['line']) _Extra = collections.namedtuple('Extra', ['name', 'content']) _extras_grammar = """ ini = (line*:p extras?:e line*:l final:s) -> (''.join(p), e, ''.join(l+[s])) line = ~extras <(~'\\n' anything)* '\\n'> final = <(~'\\n' anything)* > extras = '[' 'e' 'x' 't' 'r' 'a' 's' ']' '\\n'+ body*:b -> b body = comment | extra comment = <'#' (~'\\n' anything)* '\\n'>:c '\\n'* -> comment(c) extra = name:n ' '* '=' line:l cont*:c '\\n'* -> extra(n, ''.join([l] + c)) name = <(anything:x ?(x not in '\\n \\t='))+> cont = ' '+ <(~'\\n' anything)* '\\n'> """ _extras_compiled = makeGrammar( _extras_grammar, {"comment": _Comment, "extra": _Extra}) Error = collections.namedtuple('Error', ['message']) File = collections.namedtuple('File', ['filename', 'content']) StdOut = collections.namedtuple('StdOut', ['message']) Verbose = collections.namedtuple('Verbose', ['message']) def extras(project): """Return a dict of extra-name:content for the extras in setup.cfg.""" if 'setup.cfg' not in project: return {} c = configparser.ConfigParser() c.readfp(io.StringIO(project['setup.cfg'])) if not c.has_section('extras'):
add = '+' ws expr2:n -> ('+', n) sub = '-' ws expr2:n -> ('-', n) mul = '*' ws expr2:n -> ('*', n) div = '/' ws expr2:n -> ('/', n) addsub = ws (add | sub) muldiv = ws (mul | div) expr = expr2:left addsub*:right -> calculate(left, right) expr2 = value:left muldiv*:right -> calculate(left, right) """ operations = { '+': operator.add, '-': operator.sub, '*': operator.mul, '/': operator.truediv, } def _calculate(start, pairs): result = start for op, value in pairs: result = operations[op](result, value) return result grammar = makeGrammar(_grammar_text, {'calculate': _calculate})
times = <'twice' | ('four' | 'five' | 'six' | 'seven') 'times'>:x -> x simple_loop = '(' command_list:seq ')' ws times:rep -> 'simple_loop', seq, rep big_loop = '*' command_list:seq '; rep from * to ' <not_new_command_or_eol*>:end -> 'big_loop', seq, end together = simple_command:x ws 'tog' -> x, 'together' tbl = simple_command:x ws 'tbl' -> x, 'tbl' command = big_loop | simple_loop | together | tbl | simple_command command_list = command:first (ws ',' ws command)*:rest -> [first] + rest row = 'rnd' 's'? ws <number ('-' number)?>:x ':' ws command_list:y '.' ws -> 'row', x, y rows = row* """ g = parsley.makeGrammar(grammar, {}) print 0, g('k2tog').command() print 1, g('123').number() print 2, g('Rnd 1: k1, yo, sl1.'.lower()).row() print 3, g('Rnd 1: (k1, yo) twice, sl1.'.lower()).row() print 4, g('*k3, sl1, k4; rep from * to last st'.lower()).big_loop() print 5, g('*(k3, sl1) twice, k4; rep from * to last st'.lower()).big_loop() print 6, g('Rnd 3: K1, *(k3, sl1) twice, k4; rep from * to last st, k1.'.lower()).row() print 7, g('Rnd 1: K1, *k4, sl1, k1, sl1, k5; rep from * to last st, k1.'.lower()).row() print 8, g('Rnd 2: K1, *k3, k2tog, yo, k1, yo, ssk, k4; rep from * to last st, k1.'.lower()).row() print 0, g('knit').command() for x in test.lower().split('\n'): print repr(x) print g(x).row()
if handler is None: return no_handler(symbol, value) return handler(value) # XXX: There needs to be a character type and a string-that-escapes-newlines # type in order to have full roundtripping. _edn_grammar_definition = open("edn.parsley").read() _unwrapped_edn = makeGrammar( _edn_grammar_definition, { "Symbol": Symbol, "Keyword": Keyword, "Vector": Vector, "TaggedValue": partial(make_tagged_value, BUILTIN_READ_HANDLERS), }, name="edn", unwrap=True, ) def _make_edn_grammar(tagged_value_handler): # XXX: At last, my pact with the dark lord is fulfilled. # # I can't find any obvious way to specify bindings at the same time as # specifying input. Making a new grammar for every set of handlers is # expensive. This hideous alternative seems to work. class _specialized_edn(_unwrapped_edn): pass