def parse(input): period = sometok("period") string = p.oneplus(sometok("string")) >> (lambda x: " ".join(x)) number = sometok("number") title = string + p.skip(period) >> RecipeTitle ingredients_start = sometok("ingredients_start") + p.skip(period) >> IngredientStart dry_measure = p.maybe(sometok("measure_type")) + sometok("dry_measure") liquid_measure = sometok("liquid_measure") mix_measure = sometok("mix_measure") # is this valid ? 'g of butter', unit w/o initial_value ingredient = p.maybe(number) + p.maybe(dry_measure | liquid_measure | mix_measure) + string >> unarg(Ingredient) ingredients = p.many(ingredient) cooking_time = p.skip(sometok("cooking_time")) + (number >> unarg(CookingTime)) + p.skip(sometok("period")) oven_temp = p.skip(sometok("oven")) + p.many(number) + p.skip(sometok("oven_temp")) >> unarg(Oven) method_start = sometok("method_start") + p.skip(period) comment = p.skip(p.many(string | period)) header = title + p.maybe(comment) instruction = (string + p.skip(period)) >> parse_instruction instructions = p.many(instruction) program = (method_start + instructions) >> unarg(MethodStart) serves = (sometok("serve") + number >> (lambda x: Serve("serve", x[1]))) + p.skip(period) ingredients_section = (ingredients_start + ingredients) >> unarg(IngredientSection) recipe = ( header + p.maybe(ingredients_section) + p.maybe(cooking_time) + p.maybe(oven_temp) + p.maybe(program) + p.maybe(serves) ) >> RecipeNode main_parser = p.oneplus(recipe) return main_parser.parse(tokenize(input))
Spec('string', r'"[^"]*"'), # '\"' escapes are ignored ] useless = ['comment', 'newline', 'space'] t = make_tokenizer(specs) return [x for x in t(str) if x.type not in useless] id = sometoks(['name', 'number', 'string']) make_graph_attr = lambda args: DefAttrs(u'graph', [Attr(*args)]) make_edge = lambda x, xs, attrs: Edge([x] + xs, attrs) node_id = memoize(id) # + maybe(port) a_list = ( id + maybe(op_('=') + id) + skip(maybe(op(','))) >> unarg(Attr)) attr_list = ( many(op_('[') + many(a_list) + op_(']')) >> flatten) attr_stmt = ( (n('graph') | n('node') | n('edge')) + attr_list >> unarg(DefAttrs)) graph_attr = id + op_('=') + id >> make_graph_attr node_stmt = node_id + attr_list >> unarg(Node) # We use a fwd() because of circular definitions like (stmt_list -> stmt -> # subgraph -> stmt_list) subgraph = fwd() edge_rhs = skip(op('->') | op('--')) + (subgraph | node_id) edge_stmt = ( (subgraph | node_id) +
def F(*xargs): (s,(f,x)) = xargs if s is None: return x try: if type(s)==float and type(x)==float: ret = f[1](s,x) else: ret = f[0](s,x) except ZeroDivisionError: return None return ret res = reduce(F, args, z) return res eval = unarg(eval_expr) number = sometok('number') >> make_number name = sometok('name') >> make_name string = sometok('string') >> make_string func = sometok('func') >> make_func value = number | name | string makeop = lambda s, f: op(s) >> const(f) add = makeop('+', (ets.add, operator.add)) sub = makeop('-', (ets.sub, operator.sub)) mul = makeop('*', (ets.mul, operator.mul)) div = makeop('/', (ets.div, operator.div))
def parse(seq): 'Sequence(Token) -> object' Host = namedtuple('Host', 'exename arch host np') LogSummary = namedtuple('LogSummary', 'host wtime stages options') def mkLevel(s): rfloat = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?' rint = r'[-+]?\d+' capture = lambda m: '(' + m + ')' within_space = lambda m: r'\s*' + m + r'\s*' cfloat, cint = [within_space(capture(m)) for m in [rfloat, rint]] x = within_space('x') m = re.match( 'Level' + cint + r'domain size \(m\)' + cfloat + x + cfloat + x + cfloat + ', num elements' + cint + x + cint + x + cint + r'\(' + cint + r'\), size \(m\)' + cfloat + x + cfloat + x + cfloat, s) return Level(*m.groups()) def mkSNESIt(s): resline, ksp = s[0], s[1] res = float(resline.strip().split()[4]) indent = len(re.match(r'^( *)(?: | \d|\d\d)\d', resline).groups()[0]) // 2 return SNESIt(indent, res, ksp) def mkKSPIt(s): return float(s.strip().split()[4]) def mkKSP(s): return KSP(reason=('UNKNOWN' if len(s) == 1 else s[1]), res=s[0]) def mkSNES(s): res = s[0] reason = s[1] indent = res[0].indent for it in res[1:]: if it.indent != indent: raise RuntimeError( 'SNES monitors changed levels, perhaps -snes_converged_reason is missing:\n\tstarted with: %s\n\tunexpected: %s' % (res[0], it)) return SNES(level=None, indent=indent, reason=s[1], res=s[0]) def mkEvent(s): s = s.split() return Event(name=s[0], count=int(s[1]), time=float(s[3]), flops=float(s[5]), Mflops=float(s[-1])) def mkStage(stageheader, events): name = re.match(r'^--- Event Stage \d+: (.*)', stageheader).groups()[0] eventdict = dict((e.name, e) for e in events) return Stage(name, eventdict) def mkOption(s): return re.match(r'^(-\w+)(?:\s+(.+))?$', s).groups() def mkRun(levels, solves, logs): for x in solves: x.level = levels[-1 - x.indent] if not logs: logs = LogSummary(host=Host('unknown', 'unknown', 'unknown', -1), wtime=[], stages=[], options=[]) exename = logs[0].host.exename arch = logs[0].host.arch host = logs[0].host.host np = logs[0].host.np wtime = logs[0].wtime stages = logs[0].stages options = logs[0].options if len(logs) > 1: meanwtime = sum(h.wtime for h in logs) / len(logs) rej = 0.15 logs = [ h for h in logs if h.wtime < (1 + rej) * meanwtime and h.wtime > (1 - rej) * meanwtime ] # Exclude outliers nlogs = len(logs) wtime = sum(h.wtime for h in logs) / nlogs for i, stage in enumerate(stages): for event in stage.events: stage.events[event]._replace(time=sum( [h.stages[i].events[event].time for h in logs]) / nlogs) stage.events[event]._replace(flops=sum( [h.stages[i].events[event].flops for h in logs]) / nlogs) stage.events[event]._replace(Mflops=sum( [h.stages[i].events[event].Mflops for h in logs]) / nlogs) return Run(levels, solves, exename, arch, host, np, wtime, stages, options) def mkHost(s): (exename, arch, host, np) = re.match( r'^(\S+) on a (\S+) named (\S+) with (\d+) processors?, by .*$', s).groups() return Host(exename, arch, host, int(np)) def mkMWTime(s): mwtime = re.match( r'^Time \(sec\):\s+(\d\.\d{3}e[-+]\d\d)\s+\d\.\d{5}\s+\d\.\d{3}e[-+]\d\d$', s).groups()[0] return float(mwtime) level = sometok('level') >> mkLevel kspit = sometok('ksp_monitor') >> mkKSPIt ksp_converged = sometok('ksp_converged') >> ( lambda s: s.strip().split()[5]) ksp_diverged = sometok('ksp_diverged') >> (lambda s: s.strip().split()[7]) ksp = many(kspit) + maybe(ksp_converged | ksp_diverged) >> mkKSP snesit = sometok('snes_monitor') + maybe(ksp) >> mkSNESIt snes_converged = sometok('snes_converged') >> ( lambda s: s.strip().split()[5]) snes_diverged = sometok('snes_diverged') >> ( lambda s: s.strip().split()[7]) snes = oneplus(snesit) + (snes_converged | snes_diverged) >> mkSNES event = sometok('event') >> mkEvent stage = sometok('stage') + many(event) >> unarg(mkStage) memory_usage = sometok('memory_usage') + many( sometok('stage')) # No plans for memory usage option_table_entry = sometok('option_table_entry') >> mkOption option_table = skip(sometok('option_table_begin')) + many( option_table_entry) + skip(sometok('option_table_end')) >> dict host = sometok('hostline') >> mkHost max_wall_time = sometok('max_wall_time') >> mkMWTime log_summary = skip(sometok('summary_begin')) + host + max_wall_time + many( stage) + skip(memory_usage) + option_table >> unarg(LogSummary) petsc_log = many(level) + many(snes) + many(log_summary) + skip( finished) >> unarg(mkRun) return petsc_log.parse(seq)