def __defineParser(self):
self.parseField = parsley.makeGrammar("""
neq = <letterOrDigit*>:field ws '!=' ws <letterOrDigit*>:value ->Field(field,value,'!=')
eq = <letterOrDigit*>:field ws '==' ws <letterOrDigit*>:value ->Field(field,value,'==')
lte = <letterOrDigit*>:field ws '<=' ws <digit*'.'?digit*>:value ->Field(field,value,'<=')
gte = <letterOrDigit*>:field ws '>=' ws <digit*'.'?digit*>:value ->Field(field,value,'>=')
lt = <letterOrDigit*>:field ws '<' ws <digit*'.'?digit*>:value ->Field(field,value,'<')
gt = <letterOrDigit*>:field ws '>' ws <digit*'.'?digit*>:value ->Field(field,value,'>')
has = <letterOrDigit*>:field ws 'has' ws <(letterOrDigit|',')*>:value ->Field(field,value,'has')
in = <letterOrDigit*>:field ws 'in' ws <(letterOrDigit|',')*>:value ->Field(field,value,'in')
hasnot = <letterOrDigit*>:field ws 'not has' ws <(letterOrDigit|',')*>:value ->Field(field,value,'hasnot')
notin = <letterOrDigit*>:field ws 'not in' ws <(letterOrDigit|',')*>:value ->Field(field,value,'notin')
fieldCondition = ws (neq | eq | lte | lt | gte |gt | has | in | hasnot | notin):evalTuple ws -> evalTuple
""", {'Field': Field})
self.parse = parsley.makeGrammar("""
neq = <letterOrDigit* ws '!=' ws letterOrDigit*>:s ->str(s)
eq = <letterOrDigit* ws '==' ws letterOrDigit*>:s ->str(s)
lte = <letterOrDigit* ws '<=' ws digit*'.'?digit*>:s->str(s)
gte = <letterOrDigit* ws '>=' ws digit*'.'?digit*>:s ->str(s)
lt = <letterOrDigit* ws '<' ws digit*'.'?digit*>:s->str(s)
gt = <letterOrDigit* ws '>' ws digit*'.'?digit*>:s ->str(s)
has = <letterOrDigit* ws 'has' ws (letterOrDigit|',')* >:s ->str(s)
in = <letterOrDigit* ws 'in' ws (letterOrDigit|',')* >:s ->str(s)
hasnot = <letterOrDigit* ws 'not has' ws (letterOrDigit|',')* >:s ->str(s)
notin = <letterOrDigit* ws 'not in' ws (letterOrDigit|',')* >:s ->str(s)
parens = ws '(' ws expr:e ws ')' ws -> e
value = parens | neq | eq | lte | lt | gte |gt | has | in | hasnot | notin
ws = ' '*
and = 'AND' ws expr3:n -> ('AND', n)
or = 'OR' ws expr3:n -> ('OR', n)
not = 'NOT' ws value:n -> ('NOT', n)
checknot = ws (value|not)
andor = ws (and | or)
expr = expr3:left andor*:right -> performOperations(left, right)
expr3 = ws checknot:right -> getVal(right)
""", {"performOperations": self.performOperations, 'getVal': self.getVal})
def __init__(self, code, by_name=False, verbose=False):
lines = [line.strip() for line in code.strip().split("\n")]
self.head = lines[0].replace(Flinterpreter.WHERE, "").strip()
body = lines[1:]
self.functions = {function[0]: function[1:]
for function in map(self.__extract_function, body)}
grammar_def = Flinterpreter.GRAMMAR
function_eval = None
if by_name:
grammar_def += Flinterpreter.ARGS_BY_NAME
function_eval = self.__evaluate_function_name
else:
grammar_def += Flinterpreter.ARGS_BY_VALUE
function_eval = self.__evaluate_function_value
fnames = " | ".join("'{}'".format(name) for name in self.functions.keys())
if len(fnames) == 0:
fnames = "'THERE ARE NO EXTRA FUNCTIONS'"
grammar_def += "fname = " + fnames
self.grammar = makeGrammar(grammar_def, {})
self.verbose = verbose
self.eval_functions = {
Operator.Constant: lambda x: x[0],
Operator.PrimitiveFunction: self.__evaluate_primitive,
Operator.IfThenElse: self.__evaluate_ifelse,
Operator.Function: function_eval,
Operator.Bool: self.__evaluate_bool,
}
def _load_parser(self, grammar: str, protocol: Protocol) -> None:
"""
Load the parser using the given grammar
:param grammar: The parser grammar
:param protocol: The protocol
:return: None
"""
self.parser = parsley.makeGrammar(
grammar, {
'punctuation': string.punctuation,
'ascii_uppercase': string.ascii_uppercase,
'ascii_lowercase': string.ascii_lowercase,
'itertools': itertools,
'Art': Art,
'ArtField': ArtField,
'Field': Field,
'RelLoc': RelLoc,
'Names': Names,
'protocol': protocol,
'Boolean': Boolean,
'Size': Size,
'ArgumentExpression': ArgumentExpression,
'MethodInvocationExpression': MethodInvocationExpression,
'ConstantExpression': ConstantExpression,
'FieldAccessExpression': FieldAccessExpression,
'ThisExpression': ThisExpression,
'IfElseExpression': IfElseExpression,
})
def scss(basedir, input, encoding="utf-8") -> Iterator[Tuple[Path, Path]]:
"""Scans a SCSS input file and its imports for `@import` statements"""
p_basedir = as_pathlib_path(basedir)
p_input = as_pathlib_path(input)
contents = read(p_basedir / p_input).decode(encoding)
parts = contents.split(";")
imports = []
def capture(arg):
imports.append(arg)
grammar = parsley.makeGrammar(
"""
string = ('"' <(~'"' anything)*>:val '"') -> capture(val)
| ("'" <(~"'" anything)*>:val "'") -> capture(val)
imports = string:first (ws ',' ws string)*
import_rule = ws '@' 'i' 'm' 'p' 'o' 'r' 't' ws imports
""", {'capture': capture})
for part in parts:
part = part.strip()
# any "@import "foo"[, "bar", ...]"
if not part.startswith("@import"): continue
grammar(part).import_rule()
for i in imports:
p = Path(i + ".scss")
yield p_basedir, p
yield from scss(p_basedir, p)
def __init__(self):
self.parser = parsley.makeGrammar(gqlGrammar, {
're': re,
'GalaxyQuery': GalaxyQuery,
'GalaxyQueryComparison': GalaxyQueryComparison,
'GalaxyQueryAnd': GalaxyQueryAnd
})
def build_protocol(self, input_str: str, name: str = None) -> Protocol:
self.protocol.set_protocol_name(name)
# load grammar
with open("input_parsers/packetlang/packetlang_grammar.txt",
"r+") as grammarFile:
grammar = grammarFile.read()
# create parser
parser = parsley.makeGrammar(
grammar, {
"ascii_letters": string.ascii_letters,
"ascii_uppercase": string.ascii_uppercase,
"ascii_lowercase": string.ascii_lowercase,
"new_bitstring": self.new_bitstring,
"new_typedef": self.new_typedef,
"new_array": self.new_array,
"new_structfield": self.new_structfield,
"new_struct": self.new_struct,
"new_parameter": self.new_parameter,
"new_func": self.new_func,
"new_constant": self.new_constant,
"new_fieldaccess": self.new_fieldaccess,
"new_this": self.new_this,
"new_methodinvocation": self.new_methodinvocation,
"build_tree": self.build_tree,
"set_pdus": self.set_pdus
})
# parse input
parser(input_str.replace(" ", "").replace("\n",
"").replace("\t",
"")).protocol()
return self.protocol
def parse_file(name):
config = ConfigFile()
bindings = {
'symbols': symbols,
'config': config,
'Statement': Statement,
'RangeStmt': RangeStmt,
'Pool': Pool,
'Subnet': Subnet,
'Class': Class,
'Subclass': Subclass,
'Group': Group,
'Host': Host,
'ConfigFile': ConfigFile,
}
with open('cyder/management/commands/lib/dhcpd_compare2/'
'dhcp.parsley') as g:
grammar = makeGrammar(g.read(), bindings)
with open(name) as f:
fStr = ''
for line in f:
if not line.startswith('group'):
line = line[:comment.match(line).start(1)]
if not line or line[-1] != '\n':
line += '\n'
fStr += line
g = grammar(fStr)
g.configFile()
return config
def grammar_factory(env={}):
return parsley.makeGrammar(
"""
dot = '.'
number = <digit+dot{0,1}digit*>:val -> float(val) if "." in val else int(val)
variable = <letter+>:var -> getattr(env, var)
attribute = exactly('$')<anything+>:attr -> int(reduce(
lambda obj, attr: getattr(obj, attr, None), [env] + attr.split('.')))
parens = '(' ws expr:e ws ')' -> e
value = number | variable | attribute | parens
add = '+' ws expr2:n -> ('+', n)
sub = '-' ws expr2:n -> ('-', n)
mul = '*' ws expr3:n -> ('*', n)
div = '/' ws expr3:n -> ('/', n)
pow = '^' ws value:n -> ('^', n)
addsub = ws (add | sub)
muldiv = ws (mul | div)
expr = expr2:left addsub*:right -> calculate(left, right)
expr2 = expr3:left muldiv*:right -> calculate(left, right)
expr3 = value:left pow*:right -> calculate(left, right)
""", {
'env': env,
'calculate': calculate
})
def parse_memory_file(payload):
"""Create a grammar that can parse the output of 'free' when called
with the -c option.
Return the results as a list of dictionaries of dictionaries """
memory_grammar = r"""
memory_reports = memory_report*
memory_report = headers_line mem_line:m buffers_line:b swap_line:s '\n'
-> dict(zip(['memory', 'buffers', 'swap'], [m, b, s]))
headers_line = ws 'total' ws 'used' ws 'free' ws 'shared' ws
'buffers' ws 'cached' '\n'
mem_line = 'Mem:' ws total:t ws used:u ws free:f ws shared:s ws
buffers:b ws cached:c '\n' -> dict([t, u, f, s, b, c])
buffers_line = '-/+ buffers/cache:' ws used:u ws free:f '\n'
-> dict([u, f])
swap_line = 'Swap:' ws total:t ws used:u ws free:f '\n'
-> dict([t, u, f])
num = <digit+>
total = num:n -> ('total', n)
used = num:n -> ('used', n)
free = num:n -> ('free', n)
shared = num:n -> ('shared', n)
buffers = num:n -> ('buffers', n)
cached = num:n -> ('cached', n)
"""
memory_parser = parsley.makeGrammar(memory_grammar, {})
memory_reports = memory_parser(payload).memory_reports()
return memory_reports
def parse(text, parameters):
Utilities.logging.info("Parsing FiniteStateMachine language...")
# definition of the grammar according to Parsley
# http://parsley.readthedocs.io/en/latest/tutorial.html
grammar = r"""
# the language is collection of transitions separated by white space
main = ( ws transition:fsm ws -> fsm)*
# a transition are two words separared by an arrow ->. Return an XML string
transition = ws word:b ws '->' ws word:e -> xml('transition',xml('begin',b,self.input.position)+xml('end',e,self.input.position),self.input.position)
# a word is a collection of letters
word = ( language | letters )
language = letters:n ws '<<' code:l '>>' ws -> miniLanguage(n, l, self.input.position)
code = <(~('>>') anything)+>
letters = <letter+>
"""
# create the language object
language = parsley.makeGrammar(grammar, {'xml': xml,
'miniLanguage': lambda x,y,z : miniLanguage(x, y, z, parameters) })
# parse the text using the grammar
parsed_xml_text = '<FiniteStateMachine>'+''.join(language(text).main())+'</FiniteStateMachine>'
# Convert xml string to xml object
parsed_xml = etree.fromstring(parsed_xml_text)
return parsed_xml, parameters
def comp(inputstr):
gr = r"""
esc = '\\' anything:x -> x
symbol = ( anything:x ?(x not in '|*?+()\\') | esc:x ) -> Node(x)
group = '(' expr:x ')' -> x
repeat = (group | symbol):x ( '*' -> Node('star', x)
| '+' -> Node('plus', x)
| '?' -> Node('qmark', x)
| -> x )
cat = ( cat:x repeat:y -> Node('cat', x, y)
| repeat )
or = ( or:x '|' cat:y -> Node('or', x, y)
| cat )
expr = or?:x -> x or Node('')
ast = expr:x -> Node('root', Node('cat', x, Node('end')))
"""
tree = AST()
grammar = parsley.makeGrammar(
gr, {'special_chars': r'|*?+()\\', 'Node': tree.node})
tree.root = grammar(inputstr).ast()
dfa = tree.dfa_dot('dfa', inputstr)
return dfa, tree.dot()
def get_parser(grammar):
return parsley.makeGrammar(grammar, {
'punctuation': string.punctuation,
'ascii_uppercase': string.ascii_uppercase,
'ascii_lowercase': string.ascii_lowercase,
'itertools': itertools,
'ExtendedDiagrams': Rfc2Xml,
'rfc': Rfc(compliant=True),
'Date': Date,
'Xref': Xref,
'TocItem': TocItem,
'Section': Section,
'List': List,
'Figure': Figure,
'Artwork': Artwork,
'T': T,
'rfc_title_docname': Rfc2Xml.rfc_title_docname,
'rfc_title_abbrev': Rfc2Xml.rfc_title_abbrev,
'front_names': Rfc2Xml.front_names,
'text_paragraph': Rfc2Xml.text_paragraph,
'sections': Rfc2Xml.sections,
'printr': Rfc2Xml.printr
})
def __init__(self, expr, metric_group=None, resolution=0.5):
def calculate(start, pairs):
result = start
for op, value in pairs:
if op == '+':
result += value
elif op == '-':
result -= value
elif op == '*':
result *= value
elif op == '/':
result /= value
return result
def repmax(reps):
return self.metric_group[reps].weight
self.expr = expr
self.metric_group = metric_group
self.resolution = resolution
self.parser = parsley.makeGrammar(
WeightExpr.grammar, {
"calculate": calculate,
'repmax': repmax,
'lb2kg': lambda n: n * 0.45359237
})
def get_grammar():
gr = """
# Declare some terminals.
marker_events_begin = LEAF
marker_events_end = LEAF
unusual_file_access = LEAF
access_sensitive_file = LEAF
internet_access = LEAF
exfil_execution = access_sensitive_file internet_access
# Looking at `arp -a` or sending intranet packet probes.
possible_network_scan = LEAF
network_scan = LEAF
# Looking at other principals on the current system, e.g. `ps`.
possible_system_scan = LEAF
system_scan = LEAF
exfil_channel = http_post_activity
| ssh_activity
exfil_format = compress_activity? encrypt_activity?
"""
# Turn abbreviated declarations of foo = LEAF into: foo = 'foo'
leaf_re = re.compile(r'^(\w+)(\s*=\s*)LEAF$', flags=re.MULTILINE)
return parsley.makeGrammar(leaf_re.sub("\\1\\2'\\1'", gr), {})
def parse_file(name):
config = ConfigFile()
bindings = {
'symbols': symbols,
'config': config,
'Statement': Statement,
'RangeStmt': RangeStmt,
'Pool': Pool,
'Subnet': Subnet,
'Class': Class,
'Subclass': Subclass,
'Group': Group,
'Host': Host,
'ConfigFile': ConfigFile,
}
with open('cyder/management/commands/lib/dhcpd_compare2/'
'dhcp.parsley') as g:
grammar = makeGrammar(g.read(), bindings)
with open(name) as f:
fStr = ''
for line in f:
if not line.startswith('group'):
line = line[:comment.match(line).start(1)]
if not line or line[-1] != '\n':
line += '\n'
fStr += line
g = grammar(fStr)
g.configFile()
return config
def traceparse(jsonData):
trace = []
def traceit(*a):
trace.append(a)
JSONParser = makeGrammar(jsonGrammar, {}, tracefunc=traceit)
return JSONParser(jsonData).top(), trace
def init():
global parser
parser = StyleParser()
parser.parser = parsley.makeGrammar(open(spyral._get_spyral_path() + 'resources/style.parsley', "r").read(),
{"string": string,
"parser": parser,
"leval": literal_eval,
"Vec2D": spyral.Vec2D})
def __init__(self, rules, root_rule, repository={}):
import parsley
Grammar.__init__(self)
repo={}
for k, v in repository.items():
repo[k]=(v, checker_factory(v))[isinstance(v, Grammar)]
self.grammar=parsley.makeGrammar(rules, repo)
self.root_rule=root_rule
def parse_spec(spec, *args, **kwargs):
interpreter = Interpreter(args, kwargs)
bindings = dict(
interpreter = interpreter,
RefChain = RefChain,
RefGraph = RefGraph)
parser = parsley.makeGrammar(grammar, bindings)
return parser(spec).spec()
def __init__(self, rules, root_rule, repository={}):
import parsley
Grammar.__init__(self)
repo = {}
for k, v in repository.items():
repo[k] = (v, checker_factory(v))[isinstance(v, Grammar)]
self.grammar = parsley.makeGrammar(rules, repo)
self.root_rule = root_rule
def __init__(self, rules, root_rule="expr", repository=None):
import parsley
Grammar.__init__(self)
repo=dict(repository or {})
for key in repo:
if isinstance(repo[key], Grammar):
repo[key] = checker_factory(repo[key])
self.grammar=parsley.makeGrammar(rules, repo)
self.root_rule=root_rule
def __init__(self, depends_string):
"""Construct a Depends instance.
:param depends_string: The string description of the requirements that
need to be satisfied. See the bindep README.rst for syntax for the
requirements list.
"""
parser = makeGrammar(grammar, {})(depends_string)
self._rules = parser.rules()
def __init__(self, depends_string):
"""Construct a Depends instance.
:param depends_string: The string description of the requirements that
need to be satisfied. See the bindep README.rst for syntax for the
requirements list.
"""
parser = makeGrammar(grammar, {})(depends_string)
self._rules = parser.rules()
def __init__(self, rules, root_rule="expr", repository=None):
import parsley
Grammar.__init__(self)
repo=dict(repository or {})
for key in repo:
if isinstance(repo[key], Grammar):
repo[key] = checker_factory(repo[key])
self.grammar=parsley.makeGrammar(rules, repo)
self.root_rule=root_rule
def __init__(self, grammar_fn=__default_grammar_fn, expose_all_rules=False):
self._grammar_fn = grammar_fn
self._grammar = parsley.makeGrammar(
open(grammar_fn, "r").read(), {
"vvhgvs": vvhgvs,
"bioutils": bioutils,
"copy": copy
})
self._logger = logging.getLogger(__name__)
self._expose_rule_functions(expose_all_rules)
def parse(f):
g = parsley.makeGrammar(
grammar_source,
{'string': string, 'Plane': Plane, 'np': np, 'Entity': Entity}
)
map_source = f.read()
entities = g(map_source).map()
return entities
def init():
"""
Initializes the Styler.
"""
global parser
parser = StyleParser()
style_file = open(spyral._get_spyral_path() + "resources/style.parsley").read()
parser.parser = parsley.makeGrammar(
style_file, {"string": string, "parser": parser, "leval": literal_eval, "Vec2D": spyral.Vec2D}
)
def make_evaluator(env):
def setname(name, value):
env[name] = value
return value
return makeGrammar(eval_grammar, {'primitive_funcs': primitive_funcs,
'env': env,
'setname': setname,
'Function': Function})
def parse_dsl(self, lang):
where = []
conjunctions = []
errors = ""
if len(lang.strip()) == 0:
return {'where_string': "", 'values': []}
try:
def ex(left, op, right):
e = [left, op, right]
where.append(e)
return (left, op, right)
def q(op, ex):
# data.append(op)
return (op, ex)
def conj(op, ex):
conjunctions.append(op)
return (op, ex)
x = parsley.makeGrammar("""
string = '"' (~'"' anything)*:s '"' -> ''.join(s)
expr = string:left ws ('='|'!='|'>'|'<'):compop ws string:right -> ex(left, compop, right)
and = 'and' ws query2:n -> conj('and', n)
or = 'or' ws query2:n -> conj('or', n)
conj = ws (and | or)
query2 = expr:e1 conj*:cnj -> q(cnj, e1)
query = query2:e1 conj*:cnj -> q(cnj, e1)
""", {"ex" : ex, "q" : q, "conj" : conj })
x(lang).query()
i = 0
max = len(conjunctions)
where_string = "where "
values = []
for ex in where:
t1, op, t2 = ex
# where_string += "lower(" + t1 + ")"
where_string += "lower(" + t1 + ")"
where_string += " " + op + "lower( %s )"
if t1 == "ip":
t2 = int(netaddr.IPAddress(t2))
values.append(t2)
if i < max:
where_string += conjunctions[i] + " "
i += 1
# print where_string
# print values
return { 'where_string' : where_string, 'values' : values}
except Exception, e:
errors = str(e)
return {'error': errors }
def make_parsley_grammar(cls):
"""Return a parsley parsing expression grammar."""
return parsley.makeGrammar(
"""
not_digit = anything:x ?(x not in "0123456789")
not_d_or_p = anything:x ?(x not in "0123456789()")
ref = not_d_or_p* '(' digit{1, 3} ')' not_d_or_p*
year = not_d_or_p* <digit{4}>:y not_d_or_p* -> int(y)
date = ref? year:y ref? -> (y, 1, 1)
""", {})
def parse(context, value):
if not value:
raise RequestParseError('Must provide a value for $filter')
try:
grammar = parsley.makeGrammar(
filter_grammar,
dict(compose=compose,
ops=gops,
get_column=colgetter(context['sqlobj'])))
except EOFError:
raise RequestParseError('Bad $filter {}'.format(value))
context['sqlobj'] = context['sqlobj'].where(grammar(value).expr())
def parse(s, extra=None, bindings=None):
if not bindings:
bindings = copy.deepcopy(DEFAULT_BINDINGS)
if extra:
bindings["extra"] = extra
global PARSER
if not PARSER:
PARSER = parsley.makeGrammar(grammar.GRAMMAR,
{"lookup": bindings.__getitem__})
return PARSER(s).specification()
def header(self, request, tag):
text = """
blah
blah
*blah*
"""
BlogGrammar = makeGrammar(blog_grammar, {"tags": tags})
#output = BlogGrammar("*Hello!*").paragraphs()
output = BlogGrammar(text).paragraphs()
return tag(output)
def _make_initial_grammar(self):
g = parsley.makeGrammar(
FeeParser.basegrammar,
{
"match": re.match,
"GlyphSelector": GlyphSelector
},
unwrap=True,
)
g.globals["parser"] = self
g.rule_callRule = callRule
g.valid_verbs = ["LoadPlugin"]
return g
def load_grammar(compile=True):
if compile:
# re/compile if necessary:
if not os.path.exists(COMPILED_GRAMMAR_FILE) or \
os.path.getmtime(GRAMMAR_FILE) > os.path.getmtime(COMPILED_GRAMMAR_FILE):
compile_grammar()
# load it like this - there doesn't seem to be a better user API available:
import kgtk.kypher.grammar_compiled as cgc
cgram = cgc.createParserClass(ometa.grammar.OMetaBase, {})
return parsley.wrapGrammar(cgram)
else:
# rebuild the grammar from scratch:
return parsley.makeGrammar(KYPHER_GRAMMAR, {})
def parse(url, context):
parserfuncts = {
'callable': functools.partial(called, context),
'item': functools.partial(item, context),
'count': functools.partial(count, context),
'value': functools.partial(value, context),
'links': functools.partial(links, context)}
grammar = parsley.makeGrammar(url_grammar, parserfuncts)
try:
grammar(url).path()
except parsley.ParseError:
raise RequestParseError('Could not parse url')
def init():
"""
Initializes the Styler.
"""
global parser
parser = StyleParser()
style_file = open(spyral._get_spyral_path() +
'resources/style.parsley').read()
parser.parser = parsley.makeGrammar(style_file,
{"string": string,
"parser": parser,
"leval": literal_eval,
"Vec2D": spyral.Vec2D})
def compiled_grammar(self):
if self._compiled_grammar is None:
self._compiled_grammar = parsley.makeGrammar(
self.requirement_grammar,
{
"lookup": self._environment_bindings().get,
"VersionRequirement": VersionRequirement,
"UrlRequirement": UrlRequirement,
"PathRequirement": PathRequirement,
"EnvironmentMarker": EnvironmentMarker,
},
)
return self._compiled_grammar
def make_token_grammar():
"""
Make the token grammar.
"""
grammar = parsley.makeGrammar(r"""
punctuation = :x ?(x in "+-*/!'#$%&\,.:<=>?@^_`;" '"') -> x
float = <'-'{0, 1} digit* '.' digit+>:ds -> float(ds)
integer = <'-'{0, 1} digit+>:ds -> int(ds)
number = float | integer
itemlist =
ws item:first (ws item)*:rest ws -> [first] + rest
| ws item:only ws -> [only]
item =
infix_rel_operator
| expr:e (ws comment)* -> e
| word:w (ws comment)* -> w
| itemlist
| '[' ws quoted_itemlist:q ws ']' -> list(q)
| '[' ws ']' -> []
| '(' itemlist:lst ')' -> tuple(lst)
| (ws comment)
quoted_itemlist =
ws quoted_item:first (ws quoted_item)*:rest ws -> [first] + rest
| ws quoted_item:only ws -> [only]
quoted_item =
infix_rel_operator
| number:n (ws comment)* -> n
| word:w (ws comment)* -> w
| <(~' ' ~'[' ~']' anything)+>:w (ws comment)* -> w
| (ws comment)
| '[' ws quoted_itemlist:q ws ']' -> list(q)
| '[' ws ']' -> []
infix_rel_operator = '=' | '<>' | '>=' | '<='
word = (word_char+):l -> ''.join(l)
word_char = (escaped_char:e -> e) | (~';' unescaped_char:u -> u)
unescaped_char = (letterOrDigit:c -> c) | (~';' punctuation:p -> p)
escaped_char = '\\' (anything:c -> c)
comment = <';' rest_of_line>:c -> Comment(c)
rest_of_line = <('\\n' | (~'\n' anything))*>
parens = '(' ws expr:e ws ')' -> e
value = (number:n ->n) | (parens:p -> p)
factor = (value:v -> v) | (word:w -> w)
add = '+' ws expr2:n -> ('+', n)
mul = '*' ws factor:n -> ('*', n)
div = '/' ws factor:n -> ('/', n)
addonly = ws add
muldiv = ws (mul | div)
expr = expr2:left addonly*:right -> calculate(left, right)
expr2 = factor:left muldiv*:right -> calculate(left, right)
""", {"calculate": calculate, "Comment": Comment})
return grammar
def make_parsley_grammar(cls):
"""Return a parsley parsing expression grammar."""
return parsley.makeGrammar(
"""
not_digit = anything:x ?(x not in "0123456789")
not_d_or_q = anything:x ?(x not in "0123456789IV()")
q_letter = ('I' | 'V'):q -> q
ref = ws '(' digit{1, 3} ')' ws | '*'
quarter = not_d_or_q* <q_letter{1, 3}>:q not_d_or_q* -> q
year = not_digit* <digit{4}>:y not_digit* -> int(y)
date = year?:y quarter?:q ref? not_d_or_q* ref? ->(y, q_to_m(q), 1)
""", {"q_to_m": cls._quarter_num_to_month})
def __init__(self,grammar_fn=__default_grammar_fn):
self._grammar_fn = grammar_fn
self._grammar = parsley.makeGrammar(open(grammar_fn,'r').read(),{'hgvs': hgvs, 'bioutils': bioutils})
# define function attributes for each grammar rule, prefixed with 'parse_'
# e.g., Parser.parse_c_interval('26+2_57-3') -> Interval(...)
#TODO: exclude built-in rules
self.rules = [ m.replace('rule_','')
for m in dir(self._grammar._grammarClass)
if m.startswith('rule_') ]
for rule_name in self.rules:
att_name = 'parse_' + rule_name
rule_fxn = self.__make_parse_rule_function(rule_name)
self.__setattr__(att_name,rule_fxn)
def generate_parser(grammarFilename):
with open(grammarFilename) as grammarFile:
return parsley.makeGrammar(
grammarFile.read(), {
"ascii_uppercase": string.ascii_uppercase,
"ascii_lowercase": string.ascii_lowercase,
"ascii_letters": string.ascii_letters,
"punctuation": string.punctuation,
"rfc": rfc,
"get_doc_series": get_doc_series,
"get_ipr_code": get_ipr_code,
"structure_subsections": structure_subsections,
"infer_toc": infer_toc,
})
def _split_with_peg(self, value, grammar):
if grammar in self.grammars:
comp_grammar = self.grammars[grammar]
else:
comp_grammar = parsley.makeGrammar(grammar, {})
self.grammars[grammar] = comp_grammar
try:
values = comp_grammar(value).values()
except:
values = []
values = [six.text_type(split_value) for split_value in values]
return pd.Series(values)
def __call__(self, logger):
if self._compiled_grammar is None:
self._compiled_grammar = makeGrammar(
self.requirement_grammar,
{
"VersionRequirement": VersionRequirement,
"UrlRequirement": UrlRequirement,
"PathRequirement": PathRequirement,
"EnvironmentMarker": EnvironmentMarker,
"logger": lambda: self._logger.get_target_logger(),
},
)
pass
self._logger.set_target_logger(logger)
yield self._compiled_grammar
def make_parsley_grammar(cls):
"""Return a parsley parsing expression grammar."""
return parsley.makeGrammar(
"""
not_digit = anything:x ?(x not in "0123456789 ")
q = not_digit* ws digit:q not_digit* ws -> int(q)
y = (ws | not_digit) <digit{2}>:y (ws | not_digit) -> y
ref = ws '(' digit{1, 3} ')' ws | '*'
date = q:q not_digit* y:y ref? ws anything{0, 3} -> (dob_year(y), q_to_m(q), 1)
""", {
"q_to_m": cls._quarter_num_to_month,
"dob_year": cls._dob_year_to_four
})
def _split_with_peg(self, value, grammar):
if grammar in self.grammars:
comp_grammar = self.grammars[grammar]
else:
comp_grammar = parsley.makeGrammar(grammar, {})
self.grammars[grammar] = comp_grammar
try:
values = comp_grammar(value).values()
except:
values = []
values = [unicode(split_value) for split_value in values]
return pd.Series(values)
def make_parsley_grammar(cls):
"""Return a parsley parsing expression grammar."""
return parsley.makeGrammar(
"""
not_digit = anything:x ?(x not in "0123456789")
y = (ws | not_digit) <digit{4}>:y (ws | not_digit) -> y
m = (ws | not_digit) <digit{1, 2}>:m (ws | not_digit) -> int(m)
y_m = y:y not_digit* m:m anything* -> (y, m, 1)
m_y = m:m not_digit* y:y anything* -> (y, m, 1)
only_m = m:m anything* -> (None, m, 1)
date = y_m | m_y | only_m
""", {})
def parse_infix(text):
if not 'infix' in _parsers:
grammar = """
subexp = '(' (token | subexp)+:ts ')' ws -> Token('SUBEXP', ts)
token = fnumber | number | pow | op | unit
expr = (subexp | token)+
number = <digit+>:ds ws -> Token('NUM', int(ds))
fnumber = <digit+>:whole '.' <digit+>:dec ws -> Token('NUM', float(whole + '.' + dec))
unit = <(letter|'_')+>:sym ws -> Token('UNIT', sym)
op = ('*' | '/'):sym ws -> Token('MUL', sym)
pow = '^' <digit+>:exp ws -> Token('POW', int(exp))
"""
_parsers['infix'] = parsley.makeGrammar(grammar, {'Token': Token})
tokenized = _parsers['infix'](text).expr()
return cobble(tokenized)
def parse_mapping(filename):
"""Parse mapping `filename`. Return parsing."""
global parsley, MAPPING_PARSER
if parsley is None:
raise NotImplementedError("Parsley parsing package must be installed.")
if MAPPING_PARSER is None:
MAPPING_PARSER = parsley.makeGrammar(MAPPING_GRAMMAR, selectors.SELECTORS)
log.verbose("Parsing", repr(filename))
filename = rmap.locate_mapping(filename)
with log.augment_exception("Parsing error in", repr(filename), exception_class=exceptions.MappingFormatError):
with open(filename) as pfile:
parser = MAPPING_PARSER(pfile.read())
header, selector, comment = parser.mapping()
return Parsing(header, selector, comment)
def make_grammar():
# TODO: tuple with one value
# TODO: make sense of the return values
x = parsley.makeGrammar(
"""
clazzChar = letterOrDigit | '.'
clazz = <clazzChar+>:x -> Class(x)
# a type or a class
item = (:x ?(x in 'TUVWXYZ') -> Type(x)
| 'None' -> Type('NoneType')
| 'unknown' -> Type('unknown')
| ('string' | 'bytestring' | 'bytes' | 'unicode'):x -> StringType(x)
| clazz:x -> x)
commaRule = ',' ws rule:rule ws -> rule
orRule = '|' ws rule:rule ws -> rule
rule = ('(' ws rule:firstRule ws commaRule*:ruleList ')' -> Tuple([firstRule] + ruleList)
| 'list[' ws rule:x ws ']' -> List(x)
| 'dict[' ws rule:first ws ',' ws rule:second ws ']' -> Dict(first, second)
| (item:one
(ws orRule+:two -> Or([one] + two)
| ws '<=' ws rule:two -> BoundedType(one, two)
| ws '[' ws rule:two ws commaRule*:ruleList ']' -> ParameterizedType(one, [two] + ruleList)
| -> one)))
expr = rule:arg (ws '->' ws rule:ret -> Function(arg, ret)
| -> arg)
""",
{
"Class": Class,
"Type": Type,
"BoundedType": BoundedType,
"Tuple": Tuple,
"StringType": StringType,
"List": List,
"Dict": Dict,
"Function": Function,
"ParameterizedType": ParameterizedType,
"Or": Or,
},
)
return x
def test_eof(self):
"""
EOF should raise a nice error.
"""
import parsley
g = parsley.makeGrammar("""
dig = '1' | '2' | '3'
bits = <dig>+
""", {})
input = '123x321'
e = self.assertRaises(ParseError, g(input).dig)
self.assertEqual(e.formatError(),
dedent("""
123x321
^
Parse error at line 1, column 1: expected EOF. trail: []
"""))
def parse(self, inputstring, document):
self.setup_parse(inputstring, document)
self.document = document
self.current_node = document
self.section_handler = _SectionHandler(document)
base = os.path.dirname(os.path.abspath(__file__))
filename = os.path.join(base, "markdown.parsley")
with open(filename) as pry_file:
self.grammar_raw = pry_file.read()
self.grammar = parsley.makeGrammar(
self.grammar_raw,
dict(builder=self),
name='Markdown'
)
self.grammar(inputstring + '\n').document()
self.finish_parse()
def grammar_factory(env={}):
return parsley.makeGrammar("""
dot = '.'
number = <digit+dot{0,1}digit*>:val -> float(val) if "." in val else int(val)
variable = <letter+>:var -> getattr(env, var)
attribute = exactly('$')<anything+>:attr -> int(reduce(
lambda obj, attr: getattr(obj, attr, None), [env] + attr.split('.')))
parens = '(' ws expr:e ws ')' -> e
value = number | variable | attribute | parens
add = '+' ws expr2:n -> ('+', n)
sub = '-' ws expr2:n -> ('-', n)
mul = '*' ws expr3:n -> ('*', n)
div = '/' ws expr3:n -> ('/', n)
pow = '^' ws value:n -> ('^', n)
addsub = ws (add | sub)
muldiv = ws (mul | div)
expr = expr2:left addsub*:right -> calculate(left, right)
expr2 = expr3:left muldiv*:right -> calculate(left, right)
expr3 = value:left pow*:right -> calculate(left, right)
""", {'env': env, 'calculate': calculate})
def __init__(self, expr, metric_group=None, resolution=0.5):
def calculate(start, pairs):
result = start
for op, value in pairs:
if op == '+':
result += value
elif op == '-':
result -= value
elif op == '*':
result *= value
elif op == '/':
result /= value
return result
def repmax(reps):
return self.metric_group[reps].weight
self.expr = expr
self.metric_group = metric_group
self.resolution = resolution
self.parser = parsley.makeGrammar(
WeightExpr.grammar,
{"calculate": calculate, 'repmax': repmax, 'lb2kg': lambda n: n * 0.45359237})
_Comment = collections.namedtuple('Comment', ['line'])
_Extra = collections.namedtuple('Extra', ['name', 'content'])
_extras_grammar = """
ini = (line*:p extras?:e line*:l final:s) -> (''.join(p), e, ''.join(l+[s]))
line = ~extras <(~'\\n' anything)* '\\n'>
final = <(~'\\n' anything)* >
extras = '[' 'e' 'x' 't' 'r' 'a' 's' ']' '\\n'+ body*:b -> b
body = comment | extra
comment = <'#' (~'\\n' anything)* '\\n'>:c '\\n'* -> comment(c)
extra = name:n ' '* '=' line:l cont*:c '\\n'* -> extra(n, ''.join([l] + c))
name = <(anything:x ?(x not in '\\n \\t='))+>
cont = ' '+ <(~'\\n' anything)* '\\n'>
"""
_extras_compiled = makeGrammar(
_extras_grammar, {"comment": _Comment, "extra": _Extra})
Error = collections.namedtuple('Error', ['message'])
File = collections.namedtuple('File', ['filename', 'content'])
StdOut = collections.namedtuple('StdOut', ['message'])
Verbose = collections.namedtuple('Verbose', ['message'])
def extras(project):
"""Return a dict of extra-name:content for the extras in setup.cfg."""
if 'setup.cfg' not in project:
return {}
c = configparser.ConfigParser()
c.readfp(io.StringIO(project['setup.cfg']))
if not c.has_section('extras'):
add = '+' ws expr2:n -> ('+', n)
sub = '-' ws expr2:n -> ('-', n)
mul = '*' ws expr2:n -> ('*', n)
div = '/' ws expr2:n -> ('/', n)
addsub = ws (add | sub)
muldiv = ws (mul | div)
expr = expr2:left addsub*:right -> calculate(left, right)
expr2 = value:left muldiv*:right -> calculate(left, right)
"""
operations = {
'+': operator.add,
'-': operator.sub,
'*': operator.mul,
'/': operator.truediv,
}
def _calculate(start, pairs):
result = start
for op, value in pairs:
result = operations[op](result, value)
return result
grammar = makeGrammar(_grammar_text, {'calculate': _calculate})
times = <'twice' | ('four' | 'five' | 'six' | 'seven') 'times'>:x -> x
simple_loop = '(' command_list:seq ')' ws times:rep -> 'simple_loop', seq, rep
big_loop = '*' command_list:seq '; rep from * to ' <not_new_command_or_eol*>:end -> 'big_loop', seq, end
together = simple_command:x ws 'tog' -> x, 'together'
tbl = simple_command:x ws 'tbl' -> x, 'tbl'
command = big_loop | simple_loop | together | tbl | simple_command
command_list = command:first (ws ',' ws command)*:rest -> [first] + rest
row = 'rnd' 's'? ws <number ('-' number)?>:x ':' ws command_list:y '.' ws -> 'row', x, y
rows = row*
"""
g = parsley.makeGrammar(grammar, {})
print 0, g('k2tog').command()
print 1, g('123').number()
print 2, g('Rnd 1: k1, yo, sl1.'.lower()).row()
print 3, g('Rnd 1: (k1, yo) twice, sl1.'.lower()).row()
print 4, g('*k3, sl1, k4; rep from * to last st'.lower()).big_loop()
print 5, g('*(k3, sl1) twice, k4; rep from * to last st'.lower()).big_loop()
print 6, g('Rnd 3: K1, *(k3, sl1) twice, k4; rep from * to last st, k1.'.lower()).row()
print 7, g('Rnd 1: K1, *k4, sl1, k1, sl1, k5; rep from * to last st, k1.'.lower()).row()
print 8, g('Rnd 2: K1, *k3, k2tog, yo, k1, yo, ssk, k4; rep from * to last st, k1.'.lower()).row()
print 0, g('knit').command()
for x in test.lower().split('\n'):
print repr(x)
print g(x).row()
if handler is None:
return no_handler(symbol, value)
return handler(value)
# XXX: There needs to be a character type and a string-that-escapes-newlines
# type in order to have full roundtripping.
_edn_grammar_definition = open("edn.parsley").read()
_unwrapped_edn = makeGrammar(
_edn_grammar_definition,
{
"Symbol": Symbol,
"Keyword": Keyword,
"Vector": Vector,
"TaggedValue": partial(make_tagged_value, BUILTIN_READ_HANDLERS),
},
name="edn",
unwrap=True,
)
def _make_edn_grammar(tagged_value_handler):
# XXX: At last, my pact with the dark lord is fulfilled.
#
# I can't find any obvious way to specify bindings at the same time as
# specifying input. Making a new grammar for every set of handlers is
# expensive. This hideous alternative seems to work.
class _specialized_edn(_unwrapped_edn):
pass