def sexp_parse(seq):
'Sequence(Token) -> object'
def make_number(n):
try:
return int(n)
except ValueError:
return float(n)
def make_name(s):
return s
number = sometok('number') >> make_number
var = sometok('name') >> make_name
atom = var | number
@with_forward_decls
def sexp():
return op_('(') + atom + many(atom|sexp) + op_(')')
@with_forward_decls
def funcapp():
return var + many(atom|sexp)
primary = sexp
return primary.parse(tokenize(seq))
def parse_instruction(spec):
string = p.oneplus(sometok("string")) >> (lambda x: " ".join(x))
ordinal = sometok("ordinal")
bowl = sometok("bowl")
the = sometok("the")
dish = sometok("dish")
to = sometok("to")
into = sometok("into")
concat = lambda list: " ".join(list)
take_i = sometok("take") + (p.oneplus(string) >> concat) + sometok("from") + sometok("refrigerator")
put_i = (
sometok("put")
+ p.skip(p.maybe(the))
+ (p.oneplus(string) >> concat)
+ p.skip(into)
+ p.maybe(ordinal | the)
+ bowl
)
liquefy_1 = sometok("liquefy") + sometok("contents") + p.maybe(ordinal) + bowl
liquefy_2 = sometok("liquefy") + (p.oneplus(string) >> concat)
liquefy_i = liquefy_1 | liquefy_2
pour_i = (
sometok("pour")
+ sometok("contents")
+ p.maybe(ordinal)
+ bowl
+ sometok("into")
+ the
+ p.maybe(ordinal)
+ dish
)
fold_i = (
sometok("fold") + p.skip(p.maybe(the)) + (p.oneplus(string) >> concat) + into + p.maybe(ordinal | the) + bowl
)
# cleanup repitition
add_i = sometok("add") + (p.oneplus(string) >> concat) + p.maybe(to + p.maybe(ordinal | the) + bowl)
remove_i = (
sometok("remove") + (p.oneplus(string) >> concat) + p.maybe(sometok("from") + p.maybe(ordinal | the) + bowl)
)
combine_i = sometok("combine") + (p.oneplus(string) >> concat) + p.maybe(into + p.maybe(ordinal | the) + bowl)
divide_i = sometok("divide") + (p.oneplus(string) >> concat) + p.maybe(into + p.maybe(ordinal | the) + bowl)
add_dry_i = sometok("add_dry") + p.maybe(to + p.maybe(ordinal | the) + bowl)
stir_1 = (
sometok("stir")
+ p.maybe(the + p.maybe(ordinal | the) + bowl)
+ sometok("for")
+ sometok("number")
+ (sometok("minute") | sometok("minutes"))
)
stir_2 = sometok("stir") + (p.oneplus(string) >> concat) + into + the + p.maybe(ordinal) + bowl
stir_i = stir_1 | stir_2
mix_i = sometok("mix") + p.maybe(the + p.maybe(ordinal) + bowl) + sometok("well")
clean_i = sometok("clean") + p.maybe(ordinal | the) + bowl
loop_start_i = (sometok("string") + p.maybe(the) + (p.oneplus(string) >> concat)) >> (lambda x: ("loop_start", x))
loop_end_i = (
sometok("string") + p.maybe(p.maybe(the) + (p.oneplus(string) >> concat)) + sometok("until") + string
) >> (lambda x: ("loop_end", x))
set_aside_i = sometok("set") >> (lambda x: (x, None))
serve_with_i = sometok("serve_with") + (p.oneplus(string) >> concat)
refrigerate_i = sometok("refrigerate") + p.maybe(
sometok("for") + sometok("number") + (sometok("hour") | sometok("hours"))
)
instruction = (
take_i
| put_i
| liquefy_i
| pour_i
| add_i
| fold_i
| remove_i
| combine_i
| divide_i
| add_dry_i
| stir_i
| mix_i
| clean_i
| loop_end_i # -| ORDER matters
| loop_start_i # -|
| set_aside_i
| serve_with_i
| refrigerate_i
) >> (lambda x: Instruction(x[0].lower().replace(" ", "_"), x[1:]))
return instruction.parse(tokenize_instruction(spec))
def parse(input):
period = sometok("period")
string = p.oneplus(sometok("string")) >> (lambda x: " ".join(x))
number = sometok("number")
title = string + p.skip(period) >> RecipeTitle
ingredients_start = sometok("ingredients_start") + p.skip(period) >> IngredientStart
dry_measure = p.maybe(sometok("measure_type")) + sometok("dry_measure")
liquid_measure = sometok("liquid_measure")
mix_measure = sometok("mix_measure")
# is this valid ? 'g of butter', unit w/o initial_value
ingredient = p.maybe(number) + p.maybe(dry_measure | liquid_measure | mix_measure) + string >> unarg(Ingredient)
ingredients = p.many(ingredient)
cooking_time = p.skip(sometok("cooking_time")) + (number >> unarg(CookingTime)) + p.skip(sometok("period"))
oven_temp = p.skip(sometok("oven")) + p.many(number) + p.skip(sometok("oven_temp")) >> unarg(Oven)
method_start = sometok("method_start") + p.skip(period)
comment = p.skip(p.many(string | period))
header = title + p.maybe(comment)
instruction = (string + p.skip(period)) >> parse_instruction
instructions = p.many(instruction)
program = (method_start + instructions) >> unarg(MethodStart)
serves = (sometok("serve") + number >> (lambda x: Serve("serve", x[1]))) + p.skip(period)
ingredients_section = (ingredients_start + ingredients) >> unarg(IngredientSection)
recipe = (
header
+ p.maybe(ingredients_section)
+ p.maybe(cooking_time)
+ p.maybe(oven_temp)
+ p.maybe(program)
+ p.maybe(serves)
) >> RecipeNode
main_parser = p.oneplus(recipe)
return main_parser.parse(tokenize(input))
return std[m.group('standard')]
else:
return unichr(int(m.group('unicode'), 16))
return re_esc.sub(sub, s)
def make_string(n):
return unescape(n[1:-1])
# JSON Grammar
null = n('null') >> const(None)
true = n('true') >> const(True)
false = n('false') >> const(False)
number = sometok('number') >> make_number
string = sometok('string') >> make_string
value = fwd()
member = string + op_(':') + value >> tuple
object = (op_('{') + maybe(member + many(op_(',') + member)) + op_('}') >>
make_object)
array = (
op_('[') + maybe(value + many(op_(',') + value)) + op_(']') >> make_array)
value.define(null | true | false | object | array | number | string)
json_text = object | array
json_file = json_text + skip(eof)
name_parser_vars(locals())
def parse(seq):
if s is None:
return x
try:
if type(s)==float and type(x)==float:
ret = f[1](s,x)
else:
ret = f[0](s,x)
except ZeroDivisionError:
return None
return ret
res = reduce(F, args, z)
return res
eval = unarg(eval_expr)
number = sometok('number') >> make_number
name = sometok('name') >> make_name
string = sometok('string') >> make_string
func = sometok('func') >> make_func
value = number | name | string
makeop = lambda s, f: op(s) >> const(f)
add = makeop('+', (ets.add, operator.add))
sub = makeop('-', (ets.sub, operator.sub))
mul = makeop('*', (ets.mul, operator.mul))
div = makeop('/', (ets.div, operator.div))
mul_op = mul | div
add_op = add | sub
def make_member(n):
assert len(n) == 2
return (n[0], n[1])
def make_object(n):
return PLObject(n)
def make_name(n):
assert len(n) == 2
return PLName(n[0], n[1])
def make_array(n):
return PLArray(n)
number = sometok('number') >> make_number
string = sometok('string') >> make_string
comment = sometok('comment') >> make_comment
name = (sometok('name') | sometok('string')) + maybe(comment) >> make_name
value = fwd()
member = name + op_('=') + value >> make_member
section = maybe(comment) + oneplus(member + op_(';')) + maybe(comment) >> make_section
object = (
op_('{') +
many(section) +
op_('}')) >> make_object
array = (
op_('(') +
many(value + op_(',')) +
op_(')')) >> make_array
}
def sub(m):
if m.group('standard') is not None:
return std[m.group('standard')]
else:
return unichr(int(m.group('unicode'), 16))
return re_esc.sub(sub, s)
def make_string(n):
return unescape(n[1:-1])
# JSON Grammar
null = n('null') >> const(None)
true = n('true') >> const(True)
false = n('false') >> const(False)
number = sometok('number') >> make_number
string = sometok('string') >> make_string
value = fwd()
member = string + op_(':') + value >> tuple
object = (
op_('{') +
maybe(member + many(op_(',') + member)) +
op_('}')
>> make_object)
array = (
op_('[') +
maybe(value + many(op_(',') + value)) +
op_(']')
>> make_array)
value.define(
null
def pure_parse(seq):
'Sequence(Token) -> object'
#http://pure-lang.googlecode.com/svn/docs/pure-syntax/pure-syntax.pdf&pli=1
# This is some beautiful code... whatever it's fast
def make_number(n):
if 'L' in n:
return long(n)
try:
return int(n)
except ValueError:
return float(n)
def make_array(n):
if n is None:
return []
else:
return [n[0]] + n[1]
def make_name(s):
return s
def make_funcapp(n):
head, tail = n
if len(tail) == 0:
return head
else:
return (head, tail)
number = sometok('number') >> make_number
var = sometok('name') >> make_name
atom = var | number
pure = forward_decl()
funcapp = forward_decl()
array = forward_decl()
expr = op_('(') + funcapp + op_(')')
funcapp.define(
var + many(expr|array|atom) >> make_funcapp
)
first = (funcapp|array|atom)
array.define(
op_('[') +
first + many(op_(',') + first) +
op_(']')
>> make_array
)
# First order objects
#first_order.define(
#funcapp |
##array |
#atom
#)
@with_forward_decls
def pure():
return (first + skip(finished)) | first
primary = pure
return primary.parse(tokenize(seq))
self.name = name.lstrip('#')
def render(self):
return self.name
class Attribute(object):
def __init__(self, (name, value)):
self.name = name
self.value = value
def render(self):
return '{s.name}="{s.value}"'.format(s=self)
eol = sometok('eol') >> Eol
space = sometok('space') >> Spaces
string = sometok("string") >> (lambda s: s[1:-1])
name = sometok('name') >> (lambda s: s)
cls = sometok('class') >> Class
attr = name + skip(sometok('eq')) + string >> Attribute
identificator = sometok('id') >> Id
tag = name + many(cls | identificator | attr | skip(space)) >> Tag
complete = many(eol | tag | string | space) + eof
def compile(pyml_text, spaces=False):
parsed = complete.parse(list(tokenizer(pyml_text)))[0]
start = True
lens = [0]
tags = [[]]
class Id(object):
def __init__(self, name):
self.name = name.lstrip('#')
def render(self):
return self.name
class Attribute(object):
def __init__(self, (name, value)):
self.name = name
self.value = value
def render(self):
return '{s.name}="{s.value}"'.format(s=self)
eol = sometok('eol') >> Eol
space = sometok('space') >> Spaces
string = sometok("string") >> (lambda s: s[1:-1])
name = sometok('name') >> (lambda s: s)
cls = sometok('class') >> Class
attr = name + skip(sometok('eq')) + string >> Attribute
identificator = sometok('id') >> Id
tag = name + many(cls|identificator|attr|skip(space)) >> Tag
complete = many(eol|tag|string|space) + eof
def compile(pyml_text, spaces=False):
parsed = complete.parse(list(tokenizer(pyml_text)))[0]
start = True
lens = [0]
tags = [[]]
def parse(seq):
'Sequence(Token) -> object'
Host = namedtuple('Host', 'exename arch host np')
LogSummary = namedtuple('LogSummary', 'host wtime stages options')
def mkLevel(s):
rfloat = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?'
rint = r'[-+]?\d+'
capture = lambda m: '(' + m + ')'
within_space = lambda m: r'\s*' + m + r'\s*'
cfloat, cint = [within_space(capture(m)) for m in [rfloat, rint]]
x = within_space('x')
m = re.match(
'Level' + cint + r'domain size \(m\)' + cfloat + x + cfloat + x +
cfloat + ', num elements' + cint + x + cint + x + cint + r'\(' +
cint + r'\), size \(m\)' + cfloat + x + cfloat + x + cfloat, s)
return Level(*m.groups())
def mkSNESIt(s):
resline, ksp = s[0], s[1]
res = float(resline.strip().split()[4])
indent = len(re.match(r'^( *)(?: | \d|\d\d)\d',
resline).groups()[0]) // 2
return SNESIt(indent, res, ksp)
def mkKSPIt(s):
return float(s.strip().split()[4])
def mkKSP(s):
return KSP(reason=('UNKNOWN' if len(s) == 1 else s[1]), res=s[0])
def mkSNES(s):
res = s[0]
reason = s[1]
indent = res[0].indent
for it in res[1:]:
if it.indent != indent:
raise RuntimeError(
'SNES monitors changed levels, perhaps -snes_converged_reason is missing:\n\tstarted with: %s\n\tunexpected: %s'
% (res[0], it))
return SNES(level=None, indent=indent, reason=s[1], res=s[0])
def mkEvent(s):
s = s.split()
return Event(name=s[0],
count=int(s[1]),
time=float(s[3]),
flops=float(s[5]),
Mflops=float(s[-1]))
def mkStage(stageheader, events):
name = re.match(r'^--- Event Stage \d+: (.*)', stageheader).groups()[0]
eventdict = dict((e.name, e) for e in events)
return Stage(name, eventdict)
def mkOption(s):
return re.match(r'^(-\w+)(?:\s+(.+))?$', s).groups()
def mkRun(levels, solves, logs):
for x in solves:
x.level = levels[-1 - x.indent]
if not logs:
logs = LogSummary(host=Host('unknown', 'unknown', 'unknown', -1),
wtime=[],
stages=[],
options=[])
exename = logs[0].host.exename
arch = logs[0].host.arch
host = logs[0].host.host
np = logs[0].host.np
wtime = logs[0].wtime
stages = logs[0].stages
options = logs[0].options
if len(logs) > 1:
meanwtime = sum(h.wtime for h in logs) / len(logs)
rej = 0.15
logs = [
h for h in logs if h.wtime < (1 + rej) *
meanwtime and h.wtime > (1 - rej) * meanwtime
] # Exclude outliers
nlogs = len(logs)
wtime = sum(h.wtime for h in logs) / nlogs
for i, stage in enumerate(stages):
for event in stage.events:
stage.events[event]._replace(time=sum(
[h.stages[i].events[event].time
for h in logs]) / nlogs)
stage.events[event]._replace(flops=sum(
[h.stages[i].events[event].flops
for h in logs]) / nlogs)
stage.events[event]._replace(Mflops=sum(
[h.stages[i].events[event].Mflops
for h in logs]) / nlogs)
return Run(levels, solves, exename, arch, host, np, wtime, stages,
options)
def mkHost(s):
(exename, arch, host, np) = re.match(
r'^(\S+) on a (\S+) named (\S+) with (\d+) processors?, by .*$',
s).groups()
return Host(exename, arch, host, int(np))
def mkMWTime(s):
mwtime = re.match(
r'^Time \(sec\):\s+(\d\.\d{3}e[-+]\d\d)\s+\d\.\d{5}\s+\d\.\d{3}e[-+]\d\d$',
s).groups()[0]
return float(mwtime)
level = sometok('level') >> mkLevel
kspit = sometok('ksp_monitor') >> mkKSPIt
ksp_converged = sometok('ksp_converged') >> (
lambda s: s.strip().split()[5])
ksp_diverged = sometok('ksp_diverged') >> (lambda s: s.strip().split()[7])
ksp = many(kspit) + maybe(ksp_converged | ksp_diverged) >> mkKSP
snesit = sometok('snes_monitor') + maybe(ksp) >> mkSNESIt
snes_converged = sometok('snes_converged') >> (
lambda s: s.strip().split()[5])
snes_diverged = sometok('snes_diverged') >> (
lambda s: s.strip().split()[7])
snes = oneplus(snesit) + (snes_converged | snes_diverged) >> mkSNES
event = sometok('event') >> mkEvent
stage = sometok('stage') + many(event) >> unarg(mkStage)
memory_usage = sometok('memory_usage') + many(
sometok('stage')) # No plans for memory usage
option_table_entry = sometok('option_table_entry') >> mkOption
option_table = skip(sometok('option_table_begin')) + many(
option_table_entry) + skip(sometok('option_table_end')) >> dict
host = sometok('hostline') >> mkHost
max_wall_time = sometok('max_wall_time') >> mkMWTime
log_summary = skip(sometok('summary_begin')) + host + max_wall_time + many(
stage) + skip(memory_usage) + option_table >> unarg(LogSummary)
petsc_log = many(level) + many(snes) + many(log_summary) + skip(
finished) >> unarg(mkRun)
return petsc_log.parse(seq)
