def parse(text): grammar_path = os.path.abspath(os.path.dirname(__file__)) with open(grammar_path + Parser.grammar_file, 'r') as grammar: parser = generator.buildParser(grammar.read()).\ parserbyname(Parser.start_prod_name) return TextTools.tag(text, parser)
def check_parse(parser, input, show_ast=False): """ >>> check_parse('equal', 'no match') Traceback (most recent call last): Exception: unmatched sequence: "match" >>> check_parse('quoted', "'Hello World'") True >>> check_parse('equal', "lvl = 'x'") True >>> check_parse('and_expr', "lvl = 'x' and logger='vm' and foo='bar'") True >>> check_parse('in_expr', "x in ('x', 'xx')", False) True >>> check_parse('expression', "x in ('x', 'xx')", False) True x>>> check_parse('expression', "x in ('x', 'xx') and x='x'", False) True """ parser = generator.buildParser(decl).parserbyname(parser) retval = TextTools.tag(input, parser) if show_ast: pprint(retval) if len(input) != retval[2]: raise Exception('unmatched sequence: "%s"' % ( input[retval[2]:])) return True
def parse_problems(problem_s): decl = open('core/problem_file.def').read() parser = generator.buildParser(decl).parserbyname('problem_file') taglist = TextTools.tag(problem_s, parser) problems = taglist[1] # helpers to traverse taglist tree def get_children(node): return node[3] def get_first_child(node): return get_children(node)[0] def get_string(node): return problem_s[node[1]:node[2]] for problem in problems: c = get_children(problem) name = get_string(get_first_child(c[0])) statement = get_string(get_first_child(c[1])) solution = get_string(get_first_child(c[2])) testcases = [] for tc in get_children(c[3]): testcases.append(get_string(tc)) signature = parse_signature(solution) print signature print name print statement print solution print testcases
def parse(parser_name, input): """ execute parse and return result as if it was a sub group. This is what normally happens. Notice that the result does not contain 'quoted' >>> parser = generator.buildParser(decl).parserbyname('quoted') >>> input = "'foo'" >>> pprint(TextTools.tag(input, parser)) (1, [('quote', 0, 1, None), ('word', 1, 4, [('alphanums', 1, 4, None)]), ('quote', 4, 5, None)], 5) >>> pprint(parse('quoted', input)) ['quoted', 0, 5, [('quote', 0, 1, None), ('word', 1, 4, [('alphanums', 1, 4, None)]), ('quote', 4, 5, None)]] """ parser = generator.buildParser(decl).parserbyname(parser_name) tags = TextTools.tag(input, parser) if tags[0] == 0 or tags[2] != len(input): raise ParsingFailedError("parsing failed") return [parser_name, 0, tags[2], tags[1]]
def check_parse(parser, input, show_ast=False): """ >>> check_parse('equal', 'no match') Traceback (most recent call last): Exception: unmatched sequence: "match" >>> check_parse('quoted', "'Hello World'") True >>> check_parse('equal', "lvl = 'x'") True >>> check_parse('and_expr', "lvl = 'x' and logger='vm' and foo='bar'") True >>> check_parse('in_expr', "x in ('x', 'xx')", False) True >>> check_parse('expression', "x in ('x', 'xx')", False) True x>>> check_parse('expression', "x in ('x', 'xx') and x='x'", False) True """ parser = generator.buildParser(decl).parserbyname(parser) retval = TextTools.tag(input, parser) if show_ast: pprint(retval) if len(input) != retval[2]: raise Exception('unmatched sequence: "%s"' % (input[retval[2]:])) return True
def state( self ): self.tagcmds = {} parser = generator.buildParser(self.grammer).parserbyname('file') r = [ id(node) for node in parser ] parser = [ self.compilestate(node) for node in parser ] tbname = [ (k, v[1] + ('%03d' % (i,)) ) for i, (k, v) in enumerate( self.tagcmds.items() ) ] tbname = dict(tbname) rr = {} for k, v in self.tagcmds.items() : if v[1] not in ('TABLE','SUBTABLE'): rr[tbname[k]] = v else : rr[tbname[k]] = (v[0],v[1],[ tbname[tn] for tn in v[2] ],v[3],v[4]) self.tagcmds = rr r = [ tbname[tn] for tn in r ] return r
def work(self, filename): """work, work""" outdir = self.args.outdir basename = os.path.splitext(os.path.basename(filename))[0] xplfile = open (filename).read() simpleparser = generator.buildParser(self.declaration).parserbyname('root') gploutput = UmXPlot(basename, outdir, debug=self.args.debug, saveit=self.args.save, force=self.args.force) if self.args.arrowsize: gploutput.arrowsize = self.args.arrowsize labelsoutput = open ( "%s/%s.labels" %(outdir,basename) , 'w') # start the work datasources = list() data = list() info("starting parsing of %s" %(filename) ) taglist = TextTools.tag(xplfile, simpleparser) currentcolor = title = xlabel = ylabel = "" # used to generate default XRange xmin = sys.maxint xmax = 0.0 for tag, beg, end, subtags in taglist[1]: # read options and labels from parse tree # convert keyword labels localcolor = 0 if tag == 'text': # default color for labels localcolor = "black" for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'keyword': label = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if xplfile[beg:beg+1] == 'l': # 'l'text position = "right" elif xplfile[beg:beg+1] == 'r': position = "left" else: position = "center" # label options # defaults printthis = True labelxoffset = 0.5 labelyoffset = 0.0 labelrotation = 0 if not self.args.fontsize: labelsize = 5 else: labelsize = self.args.fontsize-1 # special cases if label == "R": localcolor = "red" #labelsize = 3 # smaller R for macroview if self.args.microview: if self.args.rexmitpos == "right": labelxoffset = -0.15 labelyoffset = 0.7 if self.args.rexmitpos == "left": labelxoffset = -0.15 labelyoffset = -0.7 elif label == "3" or label == "2" or label == "1": printthis = False elif label == "3DUP": printthis = False labelxoffset = -1 localcolor = "#32CD32" labelrotation = 90 elif label == "SYN" or label == "RST_IN": printthis = False # edit if you want localcolor = "black" if self.args.microview: labelxoffset = -0.15 labelyoffset = 0.7 # dont print S (sack) labels elif label == "S": printthis = False # escape _ label = label.replace("_","\\\_") # write label out if printthis: labelsoutput.write('set label "\\\\fontsize{%s}{%s}\\\\selectfont %s" at %s, %s '\ '%s offset %f, %f tc rgbcolor "%s" rotate by %s\n' %(labelsize, int(round(labelsize/1.2)),label,xpoint,ypoint,position,labelyoffset,labelxoffset,localcolor,labelrotation) ) # read colors elif tag == 'color': currentcolor = xplfile[beg:end] # read l/r arrow elif tag == 'darrow': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if not localcolor: localcolor = currentcolor if ('darrow',localcolor) not in datasources: datasources.append( ('darrow',localcolor) ) data.append( ( ('darrow', localcolor), "%s %s\n" %(xpoint, ypoint) ) ) elif tag == 'harrow': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if not localcolor: localcolor = currentcolor if ('harrow',localcolor) not in datasources: datasources.append( ('harrow',localcolor) ) data.append( ( ('harrow', localcolor), "%s %s\n" %(xpoint, ypoint) ) ) # read dot elif tag == 'dot': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if not localcolor: localcolor = currentcolor if ('dot',localcolor) not in datasources: datasources.append( ('dot',localcolor) ) data.append( ( ('dot', localcolor), "%s %s\n" %(xpoint, ypoint) ) ) # diamonds elif tag == 'diamond': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if not localcolor: localcolor = currentcolor if ('diamond',localcolor) not in datasources: datasources.append( ('diamond',localcolor) ) data.append( ( ('diamond', localcolor), "%s %s\n" %(xpoint, ypoint) ) ) elif tag == 'box': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if ('box',currentcolor) not in datasources: datasources.append( ('box',currentcolor) ) data.append( ( ('box',currentcolor), "%s %s\n" %(xpoint, ypoint) ) ) elif tag == 'tick': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': xpoint = xplfile[subbeg:subend] elif subtag == 'int1': ypoint = xplfile[subbeg:subend] elif subtag == 'localcolor': localcolor = xplfile[subbeg:subend] if not localcolor: localcolor = currentcolor if ('tick',localcolor) not in datasources: datasources.append( ('tick',localcolor) ) data.append( ( ('tick',localcolor), "%s %s\n" %(xpoint, ypoint) ) ) elif tag == 'line': for subtag, subbeg, subend, subparts in subtags: if subtag == 'float1': x1point = float(xplfile[subbeg:subend]) if x1point < xmin: xmin = x1point elif subtag == 'int1': y1point = xplfile[subbeg:subend] elif subtag == 'float2': x2point = float(xplfile[subbeg:subend]) if x2point > xmax: xmax = x2point elif subtag == 'int2': y2point = xplfile[subbeg:subend] if ('line',currentcolor) not in datasources: datasources.append( ('line',currentcolor) ) data.append( ( ('line',currentcolor), "%s %s %s %s\n" %(x1point, y1point, x2point, y2point) ) ) # finish close labelsoutput.close() info("data parsing complete") # write options to gpl file # labels gploutput.setXLabel(self.xlabel) gploutput.setYLabel(self.ylabel) # range debug("XRange [%s:%s]" %(xmin,xmax) ) gploutput.setXRange("[%s:%s]" %(xmin,xmax) ) if self.args.ymax: debug("YRange [*:%s]" %self.args.ymax ) gploutput.setYRange("[*:%s]" %self.args.ymax ) # size if self.args.plotsize: gploutput.setPlotSize("%scm,%scm" \ %(self.args.plotsize[0], self.args.plotsize[1])) if self.args.fontsize: gploutput.setFontSize(self.args.fontsize) if self.args.microview: gploutput.arrowheads() #iterate over data sources (color x plottype) and write file for index,datasource in enumerate(datasources): datafilename = "%s/%s.dataset.%s.%s" %(outdir, basename, datasource[1], datasource[0]) dataoutput = open ( datafilename , 'w') wrotedata = False # iterate over all data and write approciate lines to target file for dataline in data: if dataline[0] == datasource: dataoutput.write("%s" %dataline[1] ) if (wrotedata == False): wrotedata = True dataoutput.close() # only plot datasource if wrote data, else remove target file if (wrotedata == True): gploutput.plot(outdir, basename, datasource[1], datasource[0], self.args.microview) else: os.remove(datafilename) gploutput.gplot('load "%s/%s.labels"\n' %(outdir,basename) ) gploutput.arrowheads() gploutput.save()
from billys import * from functools import * from simpleparse import generator from simpleparse.stt import TextTools parser = generator.buildParser(""" things := things* thing := alphanums, separator alphanums := [a-zA-Z0-9]+ whitespace := [ \t\r] separator := whitespace, whitespace+ / [\n] """).parserbyname('things') print 'done' taglist = TextTools.tag(""" test """, parser) print 'done2' for tag, beg, end, parts in taglist: print tag, beg, end, parts raw_input()
import os from sys import stdin, stdout, stderr from simpleparse import generator from mx.TextTools import TextTools input = stdin.read() decl = open('typographify.def').read() from typo_html import codes parser = generator.buildParser(decl).parserbyname('para') taglist = TextTools.tag(input, parser) for tag, beg, end, parts in taglist[1]: if tag == 'plain': stdout.write(input[beg:end]) elif tag == 'markup': markup = parts[0] mtag, mbeg, mend = markup[:3] start, stop = codes.get(mtag, ('<!-- unknown -->', '<!-- /unknown -->')) stdout.write(start + input[mbeg + 1:mend - 1] + stop) stderr.write('parsed %s chars of %s\n' % (taglist[-1], len(input)))
import os from sys import stdin, stdout, stderr from simpleparse import generator from mx.TextTools import TextTools input = stdin.read() decl = open('typographify.def').read() from typo_html import codes parser = generator.buildParser(decl).parserbyname('para') taglist = TextTools.tag(input, parser) for tag, beg, end, parts in taglist[1]: if tag == 'plain': stdout.write(input[beg:end]) elif tag == 'markup': markup = parts[0] mtag, mbeg, mend = markup[:3] start, stop = codes.get(mtag, ('<!-- unknown -->','<!-- /unknown -->')) stdout.write(start + input[mbeg+1:mend-1] + stop) stderr.write('parsed %s chars of %s\n' % (taglist[-1], len(input)))
print (tag,start,stop,subtags), buffer def word( self, (tag,start,stop,subtags), buffer ): print (tag,start,stop,subtags), buffer def markup( self, (tag,start,stop,subtags), buffer ): print (tag,start,stop,subtags), buffer parser = Parser(grammar) for test in tests: success, children, nextcharacter = parser.parse( test, production="para", processor=Processor()) #print success, children, nextcharacter assert success and nextcharacter==len(test) print from simpleparse import generator from simpleparse.stt.TextTools import TextTools input = tests[0] parser = generator.buildParser(grammar).parserbyname("para") taglist = TextTools.tag(input, parser) for tag, beg, end, parts in taglist[1]: if tag == 'plain': stdout.write(input[beg:end]) elif tag == 'markup': markup = parts[0] mtag, mbeg, mend = markup[:3] start, stop = codes.get(mtag, ('<!-- unknown -->','<!-- / -->')) stdout.write(start + input[mbeg+1:mend-1] + stop) stderr.write('parsed %s chars of %s\n' % (taglist[-1], len(input)))