def tokenize(self, sourcecode, filesource='<stdin>'):
"Tokenize the given string of source code."
self.errmsg = NCPTL_Error(filesource)
# Keep track of all the comments we've encountered by storing
# a mapping from line number to comment (including the initial
# hash character).
self.line2comment = {}
# Initialize the lexer.
lex.lex(module=self)
# Repeatedly invoke the lexer and return all of the tokens it produces.
self.lineno = 1
lex.input(sourcecode)
self.toklist = []
while 1:
# Acquire the next token and assign it a line number if necessary.
token = lex.token()
if not token:
break
if token.lineno < self.lineno:
token.lineno = self.lineno
# Hack: Disambiguate op_mult and star on the parser's behalf.
if token.type in ["comma", "rparen"]:
try:
if self.toklist[-1].type == "op_mult":
self.toklist[-1].type = "star"
except IndexError:
pass
# We now have one more valid token.
self.toklist.append(token)
return self.toklist
def __init__(self, options):
"Initialize the profiling module."
self.backend_name = "c_profile"
self.errmsg = NCPTL_Error(self.backend_name)
# Process any arguments we were given.
leftover_opts = []
target_backend = ""
for arg in range(0, len(options)):
profile_match = re.match(r'--profile=(.*)', options[arg])
if profile_match:
target_backend = profile_match.group(1)
elif options[arg] == "--help":
# Utilize c_generic's help-string mechanism.
import codegen_c_generic
generic_self = codegen_c_generic.NCPTL_CodeGen()
generic_self.backend_name = self.backend_name
generic_self.cmdline_options.extend([
("--profile=<string>", "Specify a backend to profile")
])
generic_self.show_help()
raise SystemExit, 0
else:
leftover_opts.append(options[arg])
if not target_backend:
self.errmsg.error_fatal(
"a target backend must be specified using --profile")
# Reparent ourselves to the profiled backend.
try:
exec("import codegen_%s" % target_backend)
exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" %
target_backend)
except:
self.errmsg.error_fatal("Unable to initialize the %s backend" %
target_backend)
try:
self.name2class[target_backend] = immediate_ancestor
except AttributeError:
self.name2class = {target_backend: immediate_ancestor}
try:
self.name2class["c_profile"].__bases__ = (immediate_ancestor, )
except KeyError:
# We're the top-level class.
self.__class__.__bases__ = (immediate_ancestor, )
self.name2class["c_profile"] = self.__class__
self.c_profile_parent = self.name2class["c_profile"].__bases__[0]
immediate_ancestor.__init__(self, leftover_opts)
self.define_eventnames = 1
self.backend_name = "c_profile + " + self.backend_name
self.backend_desc = "event profiler atop " + self.backend_desc
def __init__(self, options=None):
"Initialize the LibSea code generation module."
self.errmsg = NCPTL_Error() # Placeholder until generate is called
# Process any arguments we were given.
self.source_truncate = 100 # Truncate node source code after this many characters
for arg in range(0, len(options)):
arg_match = re.match(r'--(node-code)=(.*)', options[arg])
if arg_match:
argname, argvalue = arg_match.group(1), arg_match.group(2)
if argname == "node-code":
argvalue = int(argvalue)
if argvalue == -1:
self.source_truncate = sys.maxint
else:
self.source_truncate = argvalue
for arg in range(0, len(options)):
if options[arg] == "--help":
self.show_help()
sys.exit(0)
def __init__(self, options=None):
"Initialize the DOT code generation module."
self.errmsg = NCPTL_Error() # Placeholder until generate is called
# Process any arguments we were given.
self.dot_format = "ps" # File format that dot should generate
self.extra_dot_code = [] # Arbitrary extra code to write
self.show_attrs = 1 # 1=output AST node attributes; 0=don't
self.node_code_chars = 0 # Number of characters at which to truncate node code (0=no node code; -1=all code lines)
self.show_lines = 1 # 1=output line numbers; 0=don't
self.show_source_code = 1 # 1=show the complete source code; 0=don't
self.compress_graph = 0 # 1=save space by eliding chains; 0=show everything
for arg in range(0, len(options)):
arg_match = re.match(r'--(format|extra-dot|node-code)=(.*)', options[arg])
if arg_match:
argname, argvalue = arg_match.group(1), arg_match.group(2)
if argname == "format":
self.dot_format = argvalue
elif argname == "extra-dot":
self.extra_dot_code.append(argvalue)
elif argname == "node-code":
argvalue = int(argvalue)
if argvalue == -1:
self.node_code_chars = sys.maxint
else:
self.node_code_chars = argvalue
elif options[arg] == "--compress":
self.compress_graph = 1
elif options[arg] == "--no-attrs":
self.show_attrs = 0
elif options[arg] == "--no-lines":
self.show_lines = 0
elif options[arg] == "--no-source":
self.show_source_code = 0
elif options[arg] == "--help":
self.show_help()
sys.exit(0)
def __init__(self, options):
"Initialize the execution-trace module."
self.backend_name = "c_trace"
self.errmsg = NCPTL_Error(self.backend_name)
# Process any arguments we were given.
leftover_opts = []
target_backend = ""
self.use_curses = 0
for arg in range(0, len(options)):
trace_match = re.match(r'--trace=(.*)', options[arg])
if trace_match:
target_backend = trace_match.group(1)
elif options[arg] == "--curses":
self.use_curses = 1
elif options[arg] == "--help":
# Utilize c_generic's help-string mechanism.
import codegen_c_generic
generic_self = codegen_c_generic.NCPTL_CodeGen()
generic_self.backend_name = self.backend_name
generic_self.cmdline_options.extend([
("--trace=<string>", "Specify a backend to trace"),
("--curses",
"""Display the trace with curses instead of with
fprintf()""")
])
generic_self.show_help()
raise SystemExit, 0
else:
leftover_opts.append(options[arg])
if not target_backend:
self.errmsg.error_fatal(
"a target backend must be specified using --trace")
# Reparent ourselves to the traced backend.
try:
exec("import codegen_%s" % target_backend)
exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" %
target_backend)
except:
self.errmsg.error_fatal("Unable to initialize the %s backend" %
target_backend)
try:
self.name2class[target_backend] = immediate_ancestor
except AttributeError:
self.name2class = {target_backend: immediate_ancestor}
try:
self.name2class["c_trace"].__bases__ = (immediate_ancestor, )
except KeyError:
# We're the top-level class.
self.__class__.__bases__ = (immediate_ancestor, )
self.name2class["c_trace"] = self.__class__
self.c_trace_parent = self.name2class["c_trace"].__bases__[0]
immediate_ancestor.__init__(self, leftover_opts)
self.intercept_node_funcs(self.name2class["c_trace"])
if self.use_curses:
self.set_param("LIBS", "prepend", "-lcurses")
self.define_eventnames = 1
self.backend_name = "c_trace + " + self.backend_name
self.backend_desc = "event tracer atop " + self.backend_desc
# Add a command-line option to specify which task should be monitored.
if self.use_curses:
self.base_global_parameters.extend([
("NCPTL_TYPE_INT", "cursestask", "monitor", "M",
"Processor to monitor", "0"),
("NCPTL_TYPE_INT", "cursesdelay", "delay", "D",
"Delay in milliseconds after each screen update (0=no delay)",
"0"),
("NCPTL_TYPE_INT", "breakpoint", "breakpoint", "B",
"Source line at which to enter single-stepping mode (-1=none; 0=first event)",
"-1")
])
ld_library_path)
sys.stderr.write("# [tcsh] setenv LD_LIBRARY_PATH %s\n" %
ld_library_path)
return
oneline = dynlib_out.readline()
dynlib_out.close()
except:
pass
###########################################################################
# The program starts here.
if __name__ == "__main__":
# Prepare to issue uniform error messages.
errmsg = NCPTL_Error("ncptl")
# Set default values for our command-line parameters.
outfilename = "-"
backend = None
entirefile = None
backend_options = []
filter_list = []
execute_compile = 1
execute_link = 1
keep_ints = 0
lenient = 0
be_verbose = 1
# Determine where coNCePTuaL was installed.
try:
def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None):
"Compile an AST into a list of lines of LibSea code."
self.filesource = filesource # Input file
self.sourcecode = sourcecode # coNCePTuaL source code
self.backend_name = "libsea_graph"
self.backend_desc = "parse tree in CAIDA's LibSea graph format"
self.errmsg = NCPTL_Error(filesource)
self.next_global_ID = 0 # Next LibSea ID to assign to a node
# Write a LibSea prologue.
self.libseacode = []
if self.filesource == "<command line>":
inputfile = "the source program"
cleanfilename = "stdin_graph"
else:
inputfile = os.path.abspath(self.filesource)
cleanfilename = (re.sub(r'\W', '_',
os.path.splitext(os.path.split(inputfile)[1])[0]) +
"_graph")
self.libseacode.extend([
"#" * 78,
"# This file was generated by coNCePTuaL on %s" %
time.asctime(time.localtime(time.time())),
"# using the %s backend (%s)." %
(self.backend_name, self.backend_desc),
"# Do not modify this file; modify %s instead." % inputfile,
"#" * 78])
if self.sourcecode:
self.libseacode.extend([
"#",
"# Entire source program",
"# ---------------------"])
for oneline in string.split(string.strip(self.sourcecode), "\n"):
self.libseacode.append("# %s" % oneline)
self.libseacode.extend([
"#",
"#" * 78,
""])
# Acquire information about the graph structure.
self.assign_node_IDs(ast)
nodes = sorted(self.accumulate_nodes(ast))
nodefmtwidth = int(math.ceil(math.log10(len(nodes))))
links = sorted(self.accumulate_edges(ast))
linkfmtwidth = int(math.ceil(math.log10(len(links))))
# Produce LibSea code for the graph metadata.
self.libseacode.extend([
"Graph",
"{",
" ### metadata ###",
' @name="%s";' % os.path.splitext(os.path.basename(filesource))[0],
' @description="Parse tree for %s";' % inputfile,
" @numNodes=%d;" % len(nodes),
" @numLinks=%d;" % len(links),
" @numPaths=0;",
" @numPathLinks=0;",
""])
# Produce LibSea code for the graph structural data.
self.libseacode.extend([
" ### structural data ###",
" @links=["])
for src, dest in links[:-1]:
self.libseacode.append(" { %*d; %*d; }," % \
(linkfmtwidth, src, linkfmtwidth, dest))
self.libseacode.append(" { %*d; %*d; }" % \
(linkfmtwidth, links[-1][0],
linkfmtwidth, links[-1][1]))
self.libseacode.extend([
" ];",
" @paths=;",
""])
# Produce LibSea code for the graph attribute data.
self.libseacode.extend([
" ### attribute data ###",
" @enumerations=;",
" @attributeDefinitions=["])
self.libseacode.extend(self.format_attribute("Type", 1, nodes))
self.libseacode.extend(self.format_attribute("Attribute", 2, nodes))
self.libseacode.extend(self.format_attribute("Source_code", 3, nodes))
self.libseacode.extend(self.format_selection_attr("Is_simple_stmt",
[n[0] for n in nodes if n[1] == "simple_stmt"]))
self.libseacode.extend(self.format_selection_attr("Is_constant",
[n[0] for n in nodes if n[4]]))
self.libseacode.extend(self.format_selection_attr("Is_definition",
[n[0] for n in nodes if n[5]]))
self.libseacode.extend(self.format_selection_attr("Is_leaf",
self.accumulate_leaves(ast)))
self.libseacode.extend([
" {",
" @name=$Is_root_node;",
" @type=bool;",
" @default=|| false ||;",
" @nodeValues=[ { 0; T; } ]; # Root node",
" @linkValues=["])
for linknum in range(len(links)-1):
self.libseacode.append(" { %*d; T; }," % (linkfmtwidth, linknum))
self.libseacode.extend([
" { %*d; T; }" % (linkfmtwidth, len(links)-1),
" ];",
" @pathValues=;",
" }",
" ];",
" @qualifiers=[",
" {",
" @type=$spanning_tree;",
" @name=$Parse_tree;",
' @description="Abstract syntax tree corresponding to %s";' % inputfile,
" @attributes=[",
" { @attribute=7; @alias=$root; },",
" { @attribute=7; @alias=$tree_link; }",
" ];",
" }",
" ];",
""])
# Produce LibSea code for the remaining (unused) graph features.
self.libseacode.extend([
" ### visualization hints ###",
" ; ; ; ;",
"",
" ### interface hints ###",
" ; ; ; ; ;",
"}"])
# Return the complete LibSea graph.
return self.libseacode
def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None):
"Compile an AST into a list of lines of DOT code."
self.filesource = filesource # Input file
self.sourcecode = sourcecode # coNCePTuaL source code
self.backend_name = "dot_ast"
self.backend_desc = "parse tree in AT&T's DOT language"
self.errmsg = NCPTL_Error(filesource)
# Write a DOT prologue.
self.dotcode = []
if self.filesource == "<command line>":
inputfile = "the source program"
cleanfilename = "stdin_graph"
else:
inputfile = os.path.abspath(self.filesource)
cleanfilename = (re.sub(r'\W', '_',
os.path.splitext(os.path.split(inputfile)[1])[0]) +
"_graph")
self.dotcode.extend([
"// " + "*" * 70,
"// This file was generated by coNCePTuaL on %s" %
time.asctime(time.localtime(time.time())),
"// using the %s backend (%s)." %
(self.backend_name, self.backend_desc),
"// Do not modify this file; modify %s instead." % inputfile,
"// " + "*" * 70])
if self.sourcecode:
self.dotcode.extend([
"//",
"// Entire source program",
"// ---------------------"])
for oneline in string.split(string.strip(self.sourcecode), "\n"):
self.dotcode.append("// %s" % oneline)
self.dotcode.extend([
"",
"",
"digraph %s {" % cleanfilename,
" /* Graph defaults */",
' page = "8.5, 11";',
' size = "7.5, 10";',
" node [shape=record];",
""])
if self.extra_dot_code:
self.dotcode.append(" /* Extra code specified on the command line */")
self.dotcode.extend(map(lambda str: " %s;" % re.sub(r'[\s;]+$', "", str),
self.extra_dot_code))
self.dotcode.append("")
# Walk the AST in postorder fashion to produce a graphical parse tree.
self.dotcode.append(" /* The program-specific parse tree */")
self.dotcode.append(" subgraph cluster_parse_tree {")
self.dotcode.append(' style = "invis";')
self.nextnodenum = 1
if self.compress_graph:
self.elide_chains(ast)
self.postorder_traversal(ast)
self.dotcode.append(" }")
self.dotcode.append("")
# Optionally output the source program.
if self.show_source_code:
self.dotcode.append(" /* Program source code */")
self.dotcode.append(" subgraph cluster_source_code {")
self.dotcode.append(' style = "invis";')
lineno = 1
codelines = []
for oneline in string.split(string.rstrip(self.sourcecode), "\n"):
codelines.append("%6d) %s" %
(lineno, self.string_to_dot(oneline)))
lineno = lineno + 1
self.dotcode.append(' source_code [shape=plaintext, fontname="Courier",')
self.dotcode.append(' label="%s\\l"];' % string.join(codelines, "\\l"))
self.dotcode.append(" }")
# Write a DOT footer.
self.dotcode.append("}")
return self.dotcode
