def tokenize(self, sourcecode, filesource='<stdin>'): "Tokenize the given string of source code." self.errmsg = NCPTL_Error(filesource) # Keep track of all the comments we've encountered by storing # a mapping from line number to comment (including the initial # hash character). self.line2comment = {} # Initialize the lexer. lex.lex(module=self) # Repeatedly invoke the lexer and return all of the tokens it produces. self.lineno = 1 lex.input(sourcecode) self.toklist = [] while 1: # Acquire the next token and assign it a line number if necessary. token = lex.token() if not token: break if token.lineno < self.lineno: token.lineno = self.lineno # Hack: Disambiguate op_mult and star on the parser's behalf. if token.type in ["comma", "rparen"]: try: if self.toklist[-1].type == "op_mult": self.toklist[-1].type = "star" except IndexError: pass # We now have one more valid token. self.toklist.append(token) return self.toklist
def __init__(self, options): "Initialize the profiling module." self.backend_name = "c_profile" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" for arg in range(0, len(options)): profile_match = re.match(r'--profile=(.*)', options[arg]) if profile_match: target_backend = profile_match.group(1) elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--profile=<string>", "Specify a backend to profile") ]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal( "a target backend must be specified using --profile") # Reparent ourselves to the profiled backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_profile"].__bases__ = (immediate_ancestor, ) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor, ) self.name2class["c_profile"] = self.__class__ self.c_profile_parent = self.name2class["c_profile"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.define_eventnames = 1 self.backend_name = "c_profile + " + self.backend_name self.backend_desc = "event profiler atop " + self.backend_desc
def __init__(self, options=None): "Initialize the LibSea code generation module." self.errmsg = NCPTL_Error() # Placeholder until generate is called # Process any arguments we were given. self.source_truncate = 100 # Truncate node source code after this many characters for arg in range(0, len(options)): arg_match = re.match(r'--(node-code)=(.*)', options[arg]) if arg_match: argname, argvalue = arg_match.group(1), arg_match.group(2) if argname == "node-code": argvalue = int(argvalue) if argvalue == -1: self.source_truncate = sys.maxint else: self.source_truncate = argvalue for arg in range(0, len(options)): if options[arg] == "--help": self.show_help() sys.exit(0)
def __init__(self, options=None): "Initialize the DOT code generation module." self.errmsg = NCPTL_Error() # Placeholder until generate is called # Process any arguments we were given. self.dot_format = "ps" # File format that dot should generate self.extra_dot_code = [] # Arbitrary extra code to write self.show_attrs = 1 # 1=output AST node attributes; 0=don't self.node_code_chars = 0 # Number of characters at which to truncate node code (0=no node code; -1=all code lines) self.show_lines = 1 # 1=output line numbers; 0=don't self.show_source_code = 1 # 1=show the complete source code; 0=don't self.compress_graph = 0 # 1=save space by eliding chains; 0=show everything for arg in range(0, len(options)): arg_match = re.match(r'--(format|extra-dot|node-code)=(.*)', options[arg]) if arg_match: argname, argvalue = arg_match.group(1), arg_match.group(2) if argname == "format": self.dot_format = argvalue elif argname == "extra-dot": self.extra_dot_code.append(argvalue) elif argname == "node-code": argvalue = int(argvalue) if argvalue == -1: self.node_code_chars = sys.maxint else: self.node_code_chars = argvalue elif options[arg] == "--compress": self.compress_graph = 1 elif options[arg] == "--no-attrs": self.show_attrs = 0 elif options[arg] == "--no-lines": self.show_lines = 0 elif options[arg] == "--no-source": self.show_source_code = 0 elif options[arg] == "--help": self.show_help() sys.exit(0)
def __init__(self, options): "Initialize the execution-trace module." self.backend_name = "c_trace" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" self.use_curses = 0 for arg in range(0, len(options)): trace_match = re.match(r'--trace=(.*)', options[arg]) if trace_match: target_backend = trace_match.group(1) elif options[arg] == "--curses": self.use_curses = 1 elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--trace=<string>", "Specify a backend to trace"), ("--curses", """Display the trace with curses instead of with fprintf()""") ]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal( "a target backend must be specified using --trace") # Reparent ourselves to the traced backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_trace"].__bases__ = (immediate_ancestor, ) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor, ) self.name2class["c_trace"] = self.__class__ self.c_trace_parent = self.name2class["c_trace"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.intercept_node_funcs(self.name2class["c_trace"]) if self.use_curses: self.set_param("LIBS", "prepend", "-lcurses") self.define_eventnames = 1 self.backend_name = "c_trace + " + self.backend_name self.backend_desc = "event tracer atop " + self.backend_desc # Add a command-line option to specify which task should be monitored. if self.use_curses: self.base_global_parameters.extend([ ("NCPTL_TYPE_INT", "cursestask", "monitor", "M", "Processor to monitor", "0"), ("NCPTL_TYPE_INT", "cursesdelay", "delay", "D", "Delay in milliseconds after each screen update (0=no delay)", "0"), ("NCPTL_TYPE_INT", "breakpoint", "breakpoint", "B", "Source line at which to enter single-stepping mode (-1=none; 0=first event)", "-1") ])
ld_library_path) sys.stderr.write("# [tcsh] setenv LD_LIBRARY_PATH %s\n" % ld_library_path) return oneline = dynlib_out.readline() dynlib_out.close() except: pass ########################################################################### # The program starts here. if __name__ == "__main__": # Prepare to issue uniform error messages. errmsg = NCPTL_Error("ncptl") # Set default values for our command-line parameters. outfilename = "-" backend = None entirefile = None backend_options = [] filter_list = [] execute_compile = 1 execute_link = 1 keep_ints = 0 lenient = 0 be_verbose = 1 # Determine where coNCePTuaL was installed. try:
def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None): "Compile an AST into a list of lines of LibSea code." self.filesource = filesource # Input file self.sourcecode = sourcecode # coNCePTuaL source code self.backend_name = "libsea_graph" self.backend_desc = "parse tree in CAIDA's LibSea graph format" self.errmsg = NCPTL_Error(filesource) self.next_global_ID = 0 # Next LibSea ID to assign to a node # Write a LibSea prologue. self.libseacode = [] if self.filesource == "<command line>": inputfile = "the source program" cleanfilename = "stdin_graph" else: inputfile = os.path.abspath(self.filesource) cleanfilename = (re.sub(r'\W', '_', os.path.splitext(os.path.split(inputfile)[1])[0]) + "_graph") self.libseacode.extend([ "#" * 78, "# This file was generated by coNCePTuaL on %s" % time.asctime(time.localtime(time.time())), "# using the %s backend (%s)." % (self.backend_name, self.backend_desc), "# Do not modify this file; modify %s instead." % inputfile, "#" * 78]) if self.sourcecode: self.libseacode.extend([ "#", "# Entire source program", "# ---------------------"]) for oneline in string.split(string.strip(self.sourcecode), "\n"): self.libseacode.append("# %s" % oneline) self.libseacode.extend([ "#", "#" * 78, ""]) # Acquire information about the graph structure. self.assign_node_IDs(ast) nodes = sorted(self.accumulate_nodes(ast)) nodefmtwidth = int(math.ceil(math.log10(len(nodes)))) links = sorted(self.accumulate_edges(ast)) linkfmtwidth = int(math.ceil(math.log10(len(links)))) # Produce LibSea code for the graph metadata. self.libseacode.extend([ "Graph", "{", " ### metadata ###", ' @name="%s";' % os.path.splitext(os.path.basename(filesource))[0], ' @description="Parse tree for %s";' % inputfile, " @numNodes=%d;" % len(nodes), " @numLinks=%d;" % len(links), " @numPaths=0;", " @numPathLinks=0;", ""]) # Produce LibSea code for the graph structural data. self.libseacode.extend([ " ### structural data ###", " @links=["]) for src, dest in links[:-1]: self.libseacode.append(" { %*d; %*d; }," % \ (linkfmtwidth, src, linkfmtwidth, dest)) self.libseacode.append(" { %*d; %*d; }" % \ (linkfmtwidth, links[-1][0], linkfmtwidth, links[-1][1])) self.libseacode.extend([ " ];", " @paths=;", ""]) # Produce LibSea code for the graph attribute data. self.libseacode.extend([ " ### attribute data ###", " @enumerations=;", " @attributeDefinitions=["]) self.libseacode.extend(self.format_attribute("Type", 1, nodes)) self.libseacode.extend(self.format_attribute("Attribute", 2, nodes)) self.libseacode.extend(self.format_attribute("Source_code", 3, nodes)) self.libseacode.extend(self.format_selection_attr("Is_simple_stmt", [n[0] for n in nodes if n[1] == "simple_stmt"])) self.libseacode.extend(self.format_selection_attr("Is_constant", [n[0] for n in nodes if n[4]])) self.libseacode.extend(self.format_selection_attr("Is_definition", [n[0] for n in nodes if n[5]])) self.libseacode.extend(self.format_selection_attr("Is_leaf", self.accumulate_leaves(ast))) self.libseacode.extend([ " {", " @name=$Is_root_node;", " @type=bool;", " @default=|| false ||;", " @nodeValues=[ { 0; T; } ]; # Root node", " @linkValues=["]) for linknum in range(len(links)-1): self.libseacode.append(" { %*d; T; }," % (linkfmtwidth, linknum)) self.libseacode.extend([ " { %*d; T; }" % (linkfmtwidth, len(links)-1), " ];", " @pathValues=;", " }", " ];", " @qualifiers=[", " {", " @type=$spanning_tree;", " @name=$Parse_tree;", ' @description="Abstract syntax tree corresponding to %s";' % inputfile, " @attributes=[", " { @attribute=7; @alias=$root; },", " { @attribute=7; @alias=$tree_link; }", " ];", " }", " ];", ""]) # Produce LibSea code for the remaining (unused) graph features. self.libseacode.extend([ " ### visualization hints ###", " ; ; ; ;", "", " ### interface hints ###", " ; ; ; ; ;", "}"]) # Return the complete LibSea graph. return self.libseacode
def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None): "Compile an AST into a list of lines of DOT code." self.filesource = filesource # Input file self.sourcecode = sourcecode # coNCePTuaL source code self.backend_name = "dot_ast" self.backend_desc = "parse tree in AT&T's DOT language" self.errmsg = NCPTL_Error(filesource) # Write a DOT prologue. self.dotcode = [] if self.filesource == "<command line>": inputfile = "the source program" cleanfilename = "stdin_graph" else: inputfile = os.path.abspath(self.filesource) cleanfilename = (re.sub(r'\W', '_', os.path.splitext(os.path.split(inputfile)[1])[0]) + "_graph") self.dotcode.extend([ "// " + "*" * 70, "// This file was generated by coNCePTuaL on %s" % time.asctime(time.localtime(time.time())), "// using the %s backend (%s)." % (self.backend_name, self.backend_desc), "// Do not modify this file; modify %s instead." % inputfile, "// " + "*" * 70]) if self.sourcecode: self.dotcode.extend([ "//", "// Entire source program", "// ---------------------"]) for oneline in string.split(string.strip(self.sourcecode), "\n"): self.dotcode.append("// %s" % oneline) self.dotcode.extend([ "", "", "digraph %s {" % cleanfilename, " /* Graph defaults */", ' page = "8.5, 11";', ' size = "7.5, 10";', " node [shape=record];", ""]) if self.extra_dot_code: self.dotcode.append(" /* Extra code specified on the command line */") self.dotcode.extend(map(lambda str: " %s;" % re.sub(r'[\s;]+$', "", str), self.extra_dot_code)) self.dotcode.append("") # Walk the AST in postorder fashion to produce a graphical parse tree. self.dotcode.append(" /* The program-specific parse tree */") self.dotcode.append(" subgraph cluster_parse_tree {") self.dotcode.append(' style = "invis";') self.nextnodenum = 1 if self.compress_graph: self.elide_chains(ast) self.postorder_traversal(ast) self.dotcode.append(" }") self.dotcode.append("") # Optionally output the source program. if self.show_source_code: self.dotcode.append(" /* Program source code */") self.dotcode.append(" subgraph cluster_source_code {") self.dotcode.append(' style = "invis";') lineno = 1 codelines = [] for oneline in string.split(string.rstrip(self.sourcecode), "\n"): codelines.append("%6d) %s" % (lineno, self.string_to_dot(oneline))) lineno = lineno + 1 self.dotcode.append(' source_code [shape=plaintext, fontname="Courier",') self.dotcode.append(' label="%s\\l"];' % string.join(codelines, "\\l")) self.dotcode.append(" }") # Write a DOT footer. self.dotcode.append("}") return self.dotcode