def sexp_parse(seq): 'Sequence(Token) -> object' def make_number(n): try: return int(n) except ValueError: return float(n) def make_name(s): return s number = sometok('number') >> make_number var = sometok('name') >> make_name atom = var | number @with_forward_decls def sexp(): return op_('(') + atom + many(atom|sexp) + op_(')') @with_forward_decls def funcapp(): return var + many(atom|sexp) primary = sexp return primary.parse(tokenize(seq))
def parse_instruction(spec): string = p.oneplus(sometok("string")) >> (lambda x: " ".join(x)) ordinal = sometok("ordinal") bowl = sometok("bowl") the = sometok("the") dish = sometok("dish") to = sometok("to") into = sometok("into") concat = lambda list: " ".join(list) take_i = sometok("take") + (p.oneplus(string) >> concat) + sometok("from") + sometok("refrigerator") put_i = ( sometok("put") + p.skip(p.maybe(the)) + (p.oneplus(string) >> concat) + p.skip(into) + p.maybe(ordinal | the) + bowl ) liquefy_1 = sometok("liquefy") + sometok("contents") + p.maybe(ordinal) + bowl liquefy_2 = sometok("liquefy") + (p.oneplus(string) >> concat) liquefy_i = liquefy_1 | liquefy_2 pour_i = ( sometok("pour") + sometok("contents") + p.maybe(ordinal) + bowl + sometok("into") + the + p.maybe(ordinal) + dish ) fold_i = ( sometok("fold") + p.skip(p.maybe(the)) + (p.oneplus(string) >> concat) + into + p.maybe(ordinal | the) + bowl ) # cleanup repitition add_i = sometok("add") + (p.oneplus(string) >> concat) + p.maybe(to + p.maybe(ordinal | the) + bowl) remove_i = ( sometok("remove") + (p.oneplus(string) >> concat) + p.maybe(sometok("from") + p.maybe(ordinal | the) + bowl) ) combine_i = sometok("combine") + (p.oneplus(string) >> concat) + p.maybe(into + p.maybe(ordinal | the) + bowl) divide_i = sometok("divide") + (p.oneplus(string) >> concat) + p.maybe(into + p.maybe(ordinal | the) + bowl) add_dry_i = sometok("add_dry") + p.maybe(to + p.maybe(ordinal | the) + bowl) stir_1 = ( sometok("stir") + p.maybe(the + p.maybe(ordinal | the) + bowl) + sometok("for") + sometok("number") + (sometok("minute") | sometok("minutes")) ) stir_2 = sometok("stir") + (p.oneplus(string) >> concat) + into + the + p.maybe(ordinal) + bowl stir_i = stir_1 | stir_2 mix_i = sometok("mix") + p.maybe(the + p.maybe(ordinal) + bowl) + sometok("well") clean_i = sometok("clean") + p.maybe(ordinal | the) + bowl loop_start_i = (sometok("string") + p.maybe(the) + (p.oneplus(string) >> concat)) >> (lambda x: ("loop_start", x)) loop_end_i = ( sometok("string") + p.maybe(p.maybe(the) + (p.oneplus(string) >> concat)) + sometok("until") + string ) >> (lambda x: ("loop_end", x)) set_aside_i = sometok("set") >> (lambda x: (x, None)) serve_with_i = sometok("serve_with") + (p.oneplus(string) >> concat) refrigerate_i = sometok("refrigerate") + p.maybe( sometok("for") + sometok("number") + (sometok("hour") | sometok("hours")) ) instruction = ( take_i | put_i | liquefy_i | pour_i | add_i | fold_i | remove_i | combine_i | divide_i | add_dry_i | stir_i | mix_i | clean_i | loop_end_i # -| ORDER matters | loop_start_i # -| | set_aside_i | serve_with_i | refrigerate_i ) >> (lambda x: Instruction(x[0].lower().replace(" ", "_"), x[1:])) return instruction.parse(tokenize_instruction(spec))
def parse(input): period = sometok("period") string = p.oneplus(sometok("string")) >> (lambda x: " ".join(x)) number = sometok("number") title = string + p.skip(period) >> RecipeTitle ingredients_start = sometok("ingredients_start") + p.skip(period) >> IngredientStart dry_measure = p.maybe(sometok("measure_type")) + sometok("dry_measure") liquid_measure = sometok("liquid_measure") mix_measure = sometok("mix_measure") # is this valid ? 'g of butter', unit w/o initial_value ingredient = p.maybe(number) + p.maybe(dry_measure | liquid_measure | mix_measure) + string >> unarg(Ingredient) ingredients = p.many(ingredient) cooking_time = p.skip(sometok("cooking_time")) + (number >> unarg(CookingTime)) + p.skip(sometok("period")) oven_temp = p.skip(sometok("oven")) + p.many(number) + p.skip(sometok("oven_temp")) >> unarg(Oven) method_start = sometok("method_start") + p.skip(period) comment = p.skip(p.many(string | period)) header = title + p.maybe(comment) instruction = (string + p.skip(period)) >> parse_instruction instructions = p.many(instruction) program = (method_start + instructions) >> unarg(MethodStart) serves = (sometok("serve") + number >> (lambda x: Serve("serve", x[1]))) + p.skip(period) ingredients_section = (ingredients_start + ingredients) >> unarg(IngredientSection) recipe = ( header + p.maybe(ingredients_section) + p.maybe(cooking_time) + p.maybe(oven_temp) + p.maybe(program) + p.maybe(serves) ) >> RecipeNode main_parser = p.oneplus(recipe) return main_parser.parse(tokenize(input))
return std[m.group('standard')] else: return unichr(int(m.group('unicode'), 16)) return re_esc.sub(sub, s) def make_string(n): return unescape(n[1:-1]) # JSON Grammar null = n('null') >> const(None) true = n('true') >> const(True) false = n('false') >> const(False) number = sometok('number') >> make_number string = sometok('string') >> make_string value = fwd() member = string + op_(':') + value >> tuple object = (op_('{') + maybe(member + many(op_(',') + member)) + op_('}') >> make_object) array = ( op_('[') + maybe(value + many(op_(',') + value)) + op_(']') >> make_array) value.define(null | true | false | object | array | number | string) json_text = object | array json_file = json_text + skip(eof) name_parser_vars(locals()) def parse(seq):
if s is None: return x try: if type(s)==float and type(x)==float: ret = f[1](s,x) else: ret = f[0](s,x) except ZeroDivisionError: return None return ret res = reduce(F, args, z) return res eval = unarg(eval_expr) number = sometok('number') >> make_number name = sometok('name') >> make_name string = sometok('string') >> make_string func = sometok('func') >> make_func value = number | name | string makeop = lambda s, f: op(s) >> const(f) add = makeop('+', (ets.add, operator.add)) sub = makeop('-', (ets.sub, operator.sub)) mul = makeop('*', (ets.mul, operator.mul)) div = makeop('/', (ets.div, operator.div)) mul_op = mul | div add_op = add | sub
def make_member(n): assert len(n) == 2 return (n[0], n[1]) def make_object(n): return PLObject(n) def make_name(n): assert len(n) == 2 return PLName(n[0], n[1]) def make_array(n): return PLArray(n) number = sometok('number') >> make_number string = sometok('string') >> make_string comment = sometok('comment') >> make_comment name = (sometok('name') | sometok('string')) + maybe(comment) >> make_name value = fwd() member = name + op_('=') + value >> make_member section = maybe(comment) + oneplus(member + op_(';')) + maybe(comment) >> make_section object = ( op_('{') + many(section) + op_('}')) >> make_object array = ( op_('(') + many(value + op_(',')) + op_(')')) >> make_array
} def sub(m): if m.group('standard') is not None: return std[m.group('standard')] else: return unichr(int(m.group('unicode'), 16)) return re_esc.sub(sub, s) def make_string(n): return unescape(n[1:-1]) # JSON Grammar null = n('null') >> const(None) true = n('true') >> const(True) false = n('false') >> const(False) number = sometok('number') >> make_number string = sometok('string') >> make_string value = fwd() member = string + op_(':') + value >> tuple object = ( op_('{') + maybe(member + many(op_(',') + member)) + op_('}') >> make_object) array = ( op_('[') + maybe(value + many(op_(',') + value)) + op_(']') >> make_array) value.define( null
def pure_parse(seq): 'Sequence(Token) -> object' #http://pure-lang.googlecode.com/svn/docs/pure-syntax/pure-syntax.pdf&pli=1 # This is some beautiful code... whatever it's fast def make_number(n): if 'L' in n: return long(n) try: return int(n) except ValueError: return float(n) def make_array(n): if n is None: return [] else: return [n[0]] + n[1] def make_name(s): return s def make_funcapp(n): head, tail = n if len(tail) == 0: return head else: return (head, tail) number = sometok('number') >> make_number var = sometok('name') >> make_name atom = var | number pure = forward_decl() funcapp = forward_decl() array = forward_decl() expr = op_('(') + funcapp + op_(')') funcapp.define( var + many(expr|array|atom) >> make_funcapp ) first = (funcapp|array|atom) array.define( op_('[') + first + many(op_(',') + first) + op_(']') >> make_array ) # First order objects #first_order.define( #funcapp | ##array | #atom #) @with_forward_decls def pure(): return (first + skip(finished)) | first primary = pure return primary.parse(tokenize(seq))
self.name = name.lstrip('#') def render(self): return self.name class Attribute(object): def __init__(self, (name, value)): self.name = name self.value = value def render(self): return '{s.name}="{s.value}"'.format(s=self) eol = sometok('eol') >> Eol space = sometok('space') >> Spaces string = sometok("string") >> (lambda s: s[1:-1]) name = sometok('name') >> (lambda s: s) cls = sometok('class') >> Class attr = name + skip(sometok('eq')) + string >> Attribute identificator = sometok('id') >> Id tag = name + many(cls | identificator | attr | skip(space)) >> Tag complete = many(eol | tag | string | space) + eof def compile(pyml_text, spaces=False): parsed = complete.parse(list(tokenizer(pyml_text)))[0] start = True lens = [0] tags = [[]]
class Id(object): def __init__(self, name): self.name = name.lstrip('#') def render(self): return self.name class Attribute(object): def __init__(self, (name, value)): self.name = name self.value = value def render(self): return '{s.name}="{s.value}"'.format(s=self) eol = sometok('eol') >> Eol space = sometok('space') >> Spaces string = sometok("string") >> (lambda s: s[1:-1]) name = sometok('name') >> (lambda s: s) cls = sometok('class') >> Class attr = name + skip(sometok('eq')) + string >> Attribute identificator = sometok('id') >> Id tag = name + many(cls|identificator|attr|skip(space)) >> Tag complete = many(eol|tag|string|space) + eof def compile(pyml_text, spaces=False): parsed = complete.parse(list(tokenizer(pyml_text)))[0] start = True lens = [0] tags = [[]]
def parse(seq): 'Sequence(Token) -> object' Host = namedtuple('Host', 'exename arch host np') LogSummary = namedtuple('LogSummary', 'host wtime stages options') def mkLevel(s): rfloat = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?' rint = r'[-+]?\d+' capture = lambda m: '(' + m + ')' within_space = lambda m: r'\s*' + m + r'\s*' cfloat, cint = [within_space(capture(m)) for m in [rfloat, rint]] x = within_space('x') m = re.match( 'Level' + cint + r'domain size \(m\)' + cfloat + x + cfloat + x + cfloat + ', num elements' + cint + x + cint + x + cint + r'\(' + cint + r'\), size \(m\)' + cfloat + x + cfloat + x + cfloat, s) return Level(*m.groups()) def mkSNESIt(s): resline, ksp = s[0], s[1] res = float(resline.strip().split()[4]) indent = len(re.match(r'^( *)(?: | \d|\d\d)\d', resline).groups()[0]) // 2 return SNESIt(indent, res, ksp) def mkKSPIt(s): return float(s.strip().split()[4]) def mkKSP(s): return KSP(reason=('UNKNOWN' if len(s) == 1 else s[1]), res=s[0]) def mkSNES(s): res = s[0] reason = s[1] indent = res[0].indent for it in res[1:]: if it.indent != indent: raise RuntimeError( 'SNES monitors changed levels, perhaps -snes_converged_reason is missing:\n\tstarted with: %s\n\tunexpected: %s' % (res[0], it)) return SNES(level=None, indent=indent, reason=s[1], res=s[0]) def mkEvent(s): s = s.split() return Event(name=s[0], count=int(s[1]), time=float(s[3]), flops=float(s[5]), Mflops=float(s[-1])) def mkStage(stageheader, events): name = re.match(r'^--- Event Stage \d+: (.*)', stageheader).groups()[0] eventdict = dict((e.name, e) for e in events) return Stage(name, eventdict) def mkOption(s): return re.match(r'^(-\w+)(?:\s+(.+))?$', s).groups() def mkRun(levels, solves, logs): for x in solves: x.level = levels[-1 - x.indent] if not logs: logs = LogSummary(host=Host('unknown', 'unknown', 'unknown', -1), wtime=[], stages=[], options=[]) exename = logs[0].host.exename arch = logs[0].host.arch host = logs[0].host.host np = logs[0].host.np wtime = logs[0].wtime stages = logs[0].stages options = logs[0].options if len(logs) > 1: meanwtime = sum(h.wtime for h in logs) / len(logs) rej = 0.15 logs = [ h for h in logs if h.wtime < (1 + rej) * meanwtime and h.wtime > (1 - rej) * meanwtime ] # Exclude outliers nlogs = len(logs) wtime = sum(h.wtime for h in logs) / nlogs for i, stage in enumerate(stages): for event in stage.events: stage.events[event]._replace(time=sum( [h.stages[i].events[event].time for h in logs]) / nlogs) stage.events[event]._replace(flops=sum( [h.stages[i].events[event].flops for h in logs]) / nlogs) stage.events[event]._replace(Mflops=sum( [h.stages[i].events[event].Mflops for h in logs]) / nlogs) return Run(levels, solves, exename, arch, host, np, wtime, stages, options) def mkHost(s): (exename, arch, host, np) = re.match( r'^(\S+) on a (\S+) named (\S+) with (\d+) processors?, by .*$', s).groups() return Host(exename, arch, host, int(np)) def mkMWTime(s): mwtime = re.match( r'^Time \(sec\):\s+(\d\.\d{3}e[-+]\d\d)\s+\d\.\d{5}\s+\d\.\d{3}e[-+]\d\d$', s).groups()[0] return float(mwtime) level = sometok('level') >> mkLevel kspit = sometok('ksp_monitor') >> mkKSPIt ksp_converged = sometok('ksp_converged') >> ( lambda s: s.strip().split()[5]) ksp_diverged = sometok('ksp_diverged') >> (lambda s: s.strip().split()[7]) ksp = many(kspit) + maybe(ksp_converged | ksp_diverged) >> mkKSP snesit = sometok('snes_monitor') + maybe(ksp) >> mkSNESIt snes_converged = sometok('snes_converged') >> ( lambda s: s.strip().split()[5]) snes_diverged = sometok('snes_diverged') >> ( lambda s: s.strip().split()[7]) snes = oneplus(snesit) + (snes_converged | snes_diverged) >> mkSNES event = sometok('event') >> mkEvent stage = sometok('stage') + many(event) >> unarg(mkStage) memory_usage = sometok('memory_usage') + many( sometok('stage')) # No plans for memory usage option_table_entry = sometok('option_table_entry') >> mkOption option_table = skip(sometok('option_table_begin')) + many( option_table_entry) + skip(sometok('option_table_end')) >> dict host = sometok('hostline') >> mkHost max_wall_time = sometok('max_wall_time') >> mkMWTime log_summary = skip(sometok('summary_begin')) + host + max_wall_time + many( stage) + skip(memory_usage) + option_table >> unarg(LogSummary) petsc_log = many(level) + many(snes) + many(log_summary) + skip( finished) >> unarg(mkRun) return petsc_log.parse(seq)