def tokenize(self, sourcecode, filesource='<stdin>'): "Tokenize the given string of source code." self.errmsg = NCPTL_Error(filesource) # Keep track of all the comments we've encountered by storing # a mapping from line number to comment (including the initial # hash character). self.line2comment = {} # Initialize the lexer. lex.lex(module=self) # Repeatedly invoke the lexer and return all of the tokens it produces. self.lineno = 1 lex.input(sourcecode) self.toklist = [] while 1: # Acquire the next token and assign it a line number if necessary. token = lex.token() if not token: break if token.lineno < self.lineno: token.lineno = self.lineno # Hack: Disambiguate op_mult and star on the parser's behalf. if token.type in ["comma", "rparen"]: try: if self.toklist[-1].type == "op_mult": self.toklist[-1].type = "star" except IndexError: pass # We now have one more valid token. self.toklist.append(token) return self.toklist
def __init__(self, options): "Initialize the profiling module." self.backend_name = "c_profile" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" for arg in range(0, len(options)): profile_match = re.match(r'--profile=(.*)', options[arg]) if profile_match: target_backend = profile_match.group(1) elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--profile=<string>", "Specify a backend to profile") ]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal( "a target backend must be specified using --profile") # Reparent ourselves to the profiled backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_profile"].__bases__ = (immediate_ancestor, ) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor, ) self.name2class["c_profile"] = self.__class__ self.c_profile_parent = self.name2class["c_profile"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.define_eventnames = 1 self.backend_name = "c_profile + " + self.backend_name self.backend_desc = "event profiler atop " + self.backend_desc
def __init__(self, options=None): "Initialize the LibSea code generation module." self.errmsg = NCPTL_Error() # Placeholder until generate is called # Process any arguments we were given. self.source_truncate = 100 # Truncate node source code after this many characters for arg in range(0, len(options)): arg_match = re.match(r'--(node-code)=(.*)', options[arg]) if arg_match: argname, argvalue = arg_match.group(1), arg_match.group(2) if argname == "node-code": argvalue = int(argvalue) if argvalue == -1: self.source_truncate = sys.maxint else: self.source_truncate = argvalue for arg in range(0, len(options)): if options[arg] == "--help": self.show_help() sys.exit(0)
def __init__(self, options): "Initialize the profiling module." self.backend_name = "c_profile" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" for arg in range(0, len(options)): profile_match = re.match(r'--profile=(.*)', options[arg]) if profile_match: target_backend = profile_match.group(1) elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--profile=<string>", "Specify a backend to profile")]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal("a target backend must be specified using --profile") # Reparent ourselves to the profiled backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_profile"].__bases__ = (immediate_ancestor,) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor,) self.name2class["c_profile"] = self.__class__ self.c_profile_parent = self.name2class["c_profile"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.define_eventnames = 1 self.backend_name = "c_profile + " + self.backend_name self.backend_desc = "event profiler atop " + self.backend_desc
def __init__(self, options=None): "Initialize the DOT code generation module." self.errmsg = NCPTL_Error() # Placeholder until generate is called # Process any arguments we were given. self.dot_format = "ps" # File format that dot should generate self.extra_dot_code = [] # Arbitrary extra code to write self.show_attrs = 1 # 1=output AST node attributes; 0=don't self.node_code_chars = 0 # Number of characters at which to truncate node code (0=no node code; -1=all code lines) self.show_lines = 1 # 1=output line numbers; 0=don't self.show_source_code = 1 # 1=show the complete source code; 0=don't self.compress_graph = 0 # 1=save space by eliding chains; 0=show everything for arg in range(0, len(options)): arg_match = re.match(r'--(format|extra-dot|node-code)=(.*)', options[arg]) if arg_match: argname, argvalue = arg_match.group(1), arg_match.group(2) if argname == "format": self.dot_format = argvalue elif argname == "extra-dot": self.extra_dot_code.append(argvalue) elif argname == "node-code": argvalue = int(argvalue) if argvalue == -1: self.node_code_chars = sys.maxint else: self.node_code_chars = argvalue elif options[arg] == "--compress": self.compress_graph = 1 elif options[arg] == "--no-attrs": self.show_attrs = 0 elif options[arg] == "--no-lines": self.show_lines = 0 elif options[arg] == "--no-source": self.show_source_code = 0 elif options[arg] == "--help": self.show_help() sys.exit(0)
class NCPTL_CodeGen: def __init__(self, options): "Initialize the execution-trace module." self.backend_name = "c_trace" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" self.use_curses = 0 for arg in range(0, len(options)): trace_match = re.match(r'--trace=(.*)', options[arg]) if trace_match: target_backend = trace_match.group(1) elif options[arg] == "--curses": self.use_curses = 1 elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--trace=<string>", "Specify a backend to trace"), ("--curses", """Display the trace with curses instead of with fprintf()""") ]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal( "a target backend must be specified using --trace") # Reparent ourselves to the traced backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_trace"].__bases__ = (immediate_ancestor, ) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor, ) self.name2class["c_trace"] = self.__class__ self.c_trace_parent = self.name2class["c_trace"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.intercept_node_funcs(self.name2class["c_trace"]) if self.use_curses: self.set_param("LIBS", "prepend", "-lcurses") self.define_eventnames = 1 self.backend_name = "c_trace + " + self.backend_name self.backend_desc = "event tracer atop " + self.backend_desc # Add a command-line option to specify which task should be monitored. if self.use_curses: self.base_global_parameters.extend([ ("NCPTL_TYPE_INT", "cursestask", "monitor", "M", "Processor to monitor", "0"), ("NCPTL_TYPE_INT", "cursesdelay", "delay", "D", "Delay in milliseconds after each screen update (0=no delay)", "0"), ("NCPTL_TYPE_INT", "breakpoint", "breakpoint", "B", "Source line at which to enter single-stepping mode (-1=none; 0=first event)", "-1") ]) def intercept_node_funcs(self, someclass): """ Modify all of the n_* methods (except hooks) in a class and all its parent classes so as to invoke store_node before doing anything else. """ for baseclass in someclass.__bases__: self.intercept_node_funcs(baseclass) for method_name, method_body in someclass.__dict__.items(): if self.__class__.__dict__.has_key(method_name): # The n_* methods defined in this file already do the # equivalent of store_node so there's no need to # modify them. continue if type(method_body) == types.FunctionType and re.match( r'n_[a-z_]+$', method_name): # Closure kludge -- work around Python's lack of true # closures (and lack of anything even remotely like a # closure in Python 1.5). class CloKlu: def __init__(self, trueself, method_name, method_body): self.trueself = trueself self.method_name = method_name self.method_body = method_body setattr(trueself, method_name, self.store_node) def store_node(self, node): self.trueself.current_node = node return self.method_body(self.trueself, node) CloKlu(self, method_name, method_body) # ---------------------- # # (Re)implementation of # # hook and other methods # # ---------------------- # def code_specify_include_files_POST(self, localvars): "Specify extra header files needed by the c_trace backend." includefiles = self.invoke_hook("code_specify_include_files_POST", localvars, invoke_on=self.c_trace_parent) if self.use_curses: self.push("#include <curses.h>", includefiles) return includefiles def code_declare_datatypes_EXTRA_EVENT_STATE(self, localvars): "Declare some extra tracing state to attach to each event." newdecls = [] self.code_declare_var(type="int", name="virtrank", comment="Task's current virtual rank", stack=newdecls) self.code_declare_var( type="int", name="firstline", comment="First line of source code corresponding to this event", stack=newdecls) self.code_declare_var( type="int", name="lastline", comment="Last line of source code corresponding to this event", stack=newdecls) newdecls = newdecls + self.invoke_hook( "code_declare_datatypes_EXTRA_EVENT_STATE", localvars, invoke_on=self.c_trace_parent) return newdecls def code_def_alloc_event_POST(self, localvars): "Add some tracing data to every event." return ([ "newevent->virtrank = virtrank;", "newevent->firstline = currentline[0];", "newevent->lastline = currentline[1];" ] + self.invoke_hook("code_def_alloc_event_POST", localvars, invoke_on=self.c_trace_parent)) def code_def_procev_EVENTS_DECL(self, localvars): "Declare extra variables needed within the main loop by the c_trace backend." newdecls = [] if self.use_curses: self.code_declare_var( type="static int", name="prevsrcline", rhs="-1", comment="Previously executed source-code line", stack=newdecls) return newdecls + self.invoke_hook("code_def_procev_EVENTS_DECL", localvars, invoke_on=self.c_trace_parent) def code_define_main_PRE_EVENTS(self, localvars): "Prepare curses for the main event loop." newcode = self.invoke_hook("code_define_main_PRE_EVENTS", localvars, invoke_on=self.c_trace_parent) self.push("totalevents = numevents;", newcode) if self.use_curses: self.push("if (physrank == cursestask) {", newcode) self.code_declare_var(name="numevs", comment="Mutable version of numevents", stack=newcode) self.code_declare_var(name="numtasks", comment="Mutable version of var_num_tasks", stack=newcode) self.pushmany([ "if (numevents)", "for (numevs=numevents, eventdigits=0; numevs; numevs/=10, eventdigits++)", ";", "for (numtasks=var_num_tasks-1, taskdigits=0; numtasks; numtasks/=10, taskdigits++)", ";", "(void) attrset (A_BOLD);", 'mvprintw (LINES-1, 0, "Phys: %%*s Virt: %%*s Action: %%%ds Event: %%-*s/%%-*s",' % self.event_string_len, 'taskdigits, "", taskdigits, "", "", eventdigits, "", eventdigits, "");', "(void) attrset (A_NORMAL);", 'mvprintw (LINES-1, 6, "%*d", taskdigits, physrank);', 'mvprintw (LINES-1, %d+2*taskdigits+eventdigits, "/%%*" NICS, eventdigits, numevents);' % (33 + self.event_string_len), "}" ], stack=newcode) return newcode def code_def_procev_PRE_SWITCH(self, localvars): "Output a trace message or update the screen before processing an event." newcode = [] if self.use_curses: self.event_string_len = 0 for evstr in self.events_used.keys(): if self.event_string_len < len(evstr) - 3: self.event_string_len = len(evstr) - 3 self.pushmany([ "if (physrank == cursestask) {", " /* Indicate which source-code line is currently active. */", "if (thisev->firstline-1 != prevsrcline) {", "mvchgat (prevsrcline, 6, -1, A_NORMAL, 0, NULL);", "(void) touchline (curseswin, prevsrcline, 1);", "if (thisev->firstline-1>=0 && thisev->firstline-1<LINES-1) {", "mvchgat (thisev->firstline-1, 6, -1, A_STANDOUT, 0, NULL);", "(void) touchline (curseswin, thisev->firstline-1, 1);", "}", "prevsrcline = thisev->firstline - 1;", "}", "", " /* Display other useful trace information. */", 'mvprintw (LINES-1, 14+taskdigits, "%*d", taskdigits, thisev->virtrank);', 'mvprintw (LINES-1, 24+2*taskdigits, "%%-%ds", eventnames[thisev->type]);' % self.event_string_len, 'mvprintw (LINES-1, %d+2*taskdigits, "%%*" NICS, eventdigits, i+1);' % (33 + self.event_string_len), "", " /* Update the screen and process keyboard commands. */", "(void) refresh ();", "if ((i==0 && breakpoint==0) || ((int)breakpoint==thisev->firstline)) {", " /* Enable single-stepping mode. */", "(void) nocbreak();", "(void) cbreak();", "(void) nodelay (curseswin, FALSE);", "}", "switch (getch()) {", "case 's':", "case 'S':", " /* Enable single-stepping mode. */", "(void) nocbreak();", "(void) cbreak();", "(void) nodelay (curseswin, FALSE);", "break;", "", "case ' ':", " /* Enable normal execution mode. */", "if (cursesdelay)", "(void) halfdelay ((int) ((cursesdelay + 99) / 100));", "else", "(void) nodelay (curseswin, TRUE);", "break;", "", "case 'd':", "case 'D':", " /* Delete the break point. */", "breakpoint = -1;", "break;", "", "case 'q':", "case 'Q':", " /* Quit the program. */", 'ncptl_fatal ("User interactively entered \\"Q\\" to quit the program");', "break;", "", "default:", " /* No other keys do anything special. */", "break;", "}", "}" ], stack=newcode) else: self.pushmany([ 'fprintf (stderr, "[TRACE] phys: %d | virt: %d | action: %s | event: %" NICS " / %" NICS " | lines: %d - %d\\n",', "physrank, thisev->virtrank, eventnames[thisev->type], i+1, totalevents, thisev->firstline, thisev->lastline);" ], stack=newcode) newcode = newcode + self.invoke_hook("code_define_main_PRE_SWITCH", localvars, invoke_on=self.c_trace_parent) return newcode def code_declare_globals_EXTRA(self, localvars): "Declare additional C global variables needed by the c_trace backend." newvars = [] self.code_declare_var( type="int", name="currentline", arraysize="2", comment="Current lines of source code (beginning and ending)", stack=newvars) self.code_declare_var(name="totalevents", comment="Total # of events in the event list", stack=newvars) if self.use_curses: self.code_declare_var( type="WINDOW *", name="curseswin", comment="Window to use for curses-based tracing", stack=newvars) self.code_declare_var(name="cursestask", comment="Task to trace using curses", stack=newvars) self.code_declare_var( name="cursesdelay", comment="Delay in milliseconds after each curses screen update", stack=newvars) self.code_declare_var( name="breakpoint", comment="Source line at which to enter single-stepping mode", stack=newvars) self.code_declare_var(type="int", name="eventdigits", rhs="1", comment="Number of digits in numevents", stack=newvars) self.code_declare_var(type="int", name="taskdigits", rhs="1", comment="Number of digits in var_num_tasks", stack=newvars) # Make all declarations static. static_newvars = [] for var in newvars: static_newvars.append("static " + var) # Provide a hook for including more variables. static_newvars = static_newvars + self.invoke_hook( "code_declare_globals_EXTRA", localvars, invoke_on=self.c_trace_parent) return static_newvars def code_def_init_decls_POST(self, localvars): "Declare extra variables needed within conc_initialize()." newdecls = self.invoke_hook("code_def_init_decls_POST", localvars, invoke_on=self.c_trace_parent) if self.use_curses: self.srcloop = self.code_declare_var( type="int", suffix="loop", comment="Loop over source-code lines", stack=newdecls) return newdecls def code_define_functions_INIT_COMM_3(self, localvars): "Generate code to initialize the c_trace backend." initcode = self.invoke_hook("code_define_functions_INIT_COMM_3", localvars, invoke_on=self.c_trace_parent, after=[""]) if self.use_curses: self.pushmany([ "", " /* Initialize curses. */", "if (physrank == cursestask) {", "if (!(curseswin=initscr()))", 'ncptl_fatal ("Unable to initialize the curses library");', "(void) cbreak();", "(void) noecho();", "(void) curs_set (0);", "if (cursesdelay)", "(void) halfdelay ((int)((cursesdelay+99)/100));", "else", "(void) nodelay (curseswin, TRUE);", "for (%s=0; %s<(int)(sizeof(sourcecode)/sizeof(char *))-1; %s++) {" % (self.srcloop, self.srcloop, self.srcloop), "if (%s >= LINES-1)" % self.srcloop, "break;", "(void) attrset (A_BOLD);", '(void) mvprintw (%s, 0, "%%3d. ", %s+1);' % (self.srcloop, self.srcloop), "(void) attrset (A_NORMAL);", '(void) printw ("%%.*s", COLS, sourcecode[%s]);' % self.srcloop, "}", "(void) refresh();", "}" ], stack=initcode) return initcode # Completely redefine codegen_c_generic.py's code_allocate_event method. def code_allocate_event(self, event_type, stack=None, declare="CONC_EVENT *thisev ="): "Push the code to allocate an event and keep track of used events." self.push( "%s (currentline[0]=%d, currentline[1]=%d, conc_allocate_event (%s));" % (declare, self.current_node.lineno0, self.current_node.lineno1, event_type), stack) self.events_used[event_type] = 1 def code_def_exit_handler_BODY(self, localvars): "Shut down curses if necessary." if self.use_curses: exitcode = ["if (physrank == cursestask)", "(void) endwin();"] else: exitcode = [] exitcode = exitcode + self.invoke_hook("code_def_exit_handler_BODY", localvars, invoke_on=self.c_trace_parent) return exitcode def n_outputs(self, node): "Write a message to standard out, but not if we're using curses." self.current_node = node self.c_trace_parent.n_outputs(self, node) if self.use_curses: self.arbitrary_code[-1] = []
def __init__(self, options): "Initialize the execution-trace module." self.backend_name = "c_trace" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" self.use_curses = 0 for arg in range(0, len(options)): trace_match = re.match(r'--trace=(.*)', options[arg]) if trace_match: target_backend = trace_match.group(1) elif options[arg] == "--curses": self.use_curses = 1 elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--trace=<string>", "Specify a backend to trace"), ("--curses", """Display the trace with curses instead of with fprintf()""")]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal("a target backend must be specified using --trace") # Reparent ourselves to the traced backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_trace"].__bases__ = (immediate_ancestor,) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor,) self.name2class["c_trace"] = self.__class__ self.c_trace_parent = self.name2class["c_trace"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.intercept_node_funcs(self.name2class["c_trace"]) if self.use_curses: self.set_param("LIBS", "prepend", "-lcurses") self.define_eventnames = 1 self.backend_name = "c_trace + " + self.backend_name self.backend_desc = "event tracer atop " + self.backend_desc # Add a command-line option to specify which task should be monitored. if self.use_curses: self.base_global_parameters.extend([("NCPTL_TYPE_INT", "cursestask", "monitor", "M", "Processor to monitor", "0"), ("NCPTL_TYPE_INT", "cursesdelay", "delay", "D", "Delay in milliseconds after each screen update (0=no delay)", "0"), ("NCPTL_TYPE_INT", "breakpoint", "breakpoint", "B", "Source line at which to enter single-stepping mode (-1=none; 0=first event)", "-1")])
class NCPTL_CodeGen: def __init__(self, options): "Initialize the execution-trace module." self.backend_name = "c_trace" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" self.use_curses = 0 for arg in range(0, len(options)): trace_match = re.match(r'--trace=(.*)', options[arg]) if trace_match: target_backend = trace_match.group(1) elif options[arg] == "--curses": self.use_curses = 1 elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--trace=<string>", "Specify a backend to trace"), ("--curses", """Display the trace with curses instead of with fprintf()""")]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal("a target backend must be specified using --trace") # Reparent ourselves to the traced backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_trace"].__bases__ = (immediate_ancestor,) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor,) self.name2class["c_trace"] = self.__class__ self.c_trace_parent = self.name2class["c_trace"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.intercept_node_funcs(self.name2class["c_trace"]) if self.use_curses: self.set_param("LIBS", "prepend", "-lcurses") self.define_eventnames = 1 self.backend_name = "c_trace + " + self.backend_name self.backend_desc = "event tracer atop " + self.backend_desc # Add a command-line option to specify which task should be monitored. if self.use_curses: self.base_global_parameters.extend([("NCPTL_TYPE_INT", "cursestask", "monitor", "M", "Processor to monitor", "0"), ("NCPTL_TYPE_INT", "cursesdelay", "delay", "D", "Delay in milliseconds after each screen update (0=no delay)", "0"), ("NCPTL_TYPE_INT", "breakpoint", "breakpoint", "B", "Source line at which to enter single-stepping mode (-1=none; 0=first event)", "-1")]) def intercept_node_funcs(self, someclass): """ Modify all of the n_* methods (except hooks) in a class and all its parent classes so as to invoke store_node before doing anything else. """ for baseclass in someclass.__bases__: self.intercept_node_funcs(baseclass) for method_name, method_body in someclass.__dict__.items(): if self.__class__.__dict__.has_key(method_name): # The n_* methods defined in this file already do the # equivalent of store_node so there's no need to # modify them. continue if type(method_body)==types.FunctionType and re.match(r'n_[a-z_]+$', method_name): # Closure kludge -- work around Python's lack of true # closures (and lack of anything even remotely like a # closure in Python 1.5). class CloKlu: def __init__(self, trueself, method_name, method_body): self.trueself = trueself self.method_name = method_name self.method_body = method_body setattr(trueself, method_name, self.store_node) def store_node(self, node): self.trueself.current_node = node return self.method_body(self.trueself, node) CloKlu(self, method_name, method_body) # ---------------------- # # (Re)implementation of # # hook and other methods # # ---------------------- # def code_specify_include_files_POST(self, localvars): "Specify extra header files needed by the c_trace backend." includefiles = self.invoke_hook("code_specify_include_files_POST", localvars, invoke_on=self.c_trace_parent) if self.use_curses: self.push("#include <curses.h>", includefiles) return includefiles def code_declare_datatypes_EXTRA_EVENT_STATE(self, localvars): "Declare some extra tracing state to attach to each event." newdecls = [] self.code_declare_var(type="int", name="virtrank", comment="Task's current virtual rank", stack=newdecls) self.code_declare_var(type="int", name="firstline", comment="First line of source code corresponding to this event", stack=newdecls) self.code_declare_var(type="int", name="lastline", comment="Last line of source code corresponding to this event", stack=newdecls) newdecls = newdecls + self.invoke_hook("code_declare_datatypes_EXTRA_EVENT_STATE", localvars, invoke_on=self.c_trace_parent) return newdecls def code_def_alloc_event_POST(self, localvars): "Add some tracing data to every event." return ([ "newevent->virtrank = virtrank;", "newevent->firstline = currentline[0];", "newevent->lastline = currentline[1];"] + self.invoke_hook("code_def_alloc_event_POST", localvars, invoke_on=self.c_trace_parent)) def code_def_procev_EVENTS_DECL(self, localvars): "Declare extra variables needed within the main loop by the c_trace backend." newdecls = [] if self.use_curses: self.code_declare_var(type="static int", name="prevsrcline", rhs="-1", comment="Previously executed source-code line", stack=newdecls) return newdecls + self.invoke_hook("code_def_procev_EVENTS_DECL", localvars, invoke_on=self.c_trace_parent) def code_define_main_PRE_EVENTS(self, localvars): "Prepare curses for the main event loop." newcode = self.invoke_hook("code_define_main_PRE_EVENTS", localvars, invoke_on=self.c_trace_parent) self.push("totalevents = numevents;", newcode); if self.use_curses: self.push("if (physrank == cursestask) {", newcode) self.code_declare_var(name="numevs", comment="Mutable version of numevents", stack=newcode) self.code_declare_var(name="numtasks", comment="Mutable version of var_num_tasks", stack=newcode) self.pushmany([ "if (numevents)", "for (numevs=numevents, eventdigits=0; numevs; numevs/=10, eventdigits++)", ";", "for (numtasks=var_num_tasks-1, taskdigits=0; numtasks; numtasks/=10, taskdigits++)", ";", "(void) attrset (A_BOLD);", 'mvprintw (LINES-1, 0, "Phys: %%*s Virt: %%*s Action: %%%ds Event: %%-*s/%%-*s",' % self.event_string_len, 'taskdigits, "", taskdigits, "", "", eventdigits, "", eventdigits, "");', "(void) attrset (A_NORMAL);", 'mvprintw (LINES-1, 6, "%*d", taskdigits, physrank);', 'mvprintw (LINES-1, %d+2*taskdigits+eventdigits, "/%%*" NICS, eventdigits, numevents);' % (33+self.event_string_len), "}"], stack=newcode) return newcode def code_def_procev_PRE_SWITCH(self, localvars): "Output a trace message or update the screen before processing an event." newcode = [] if self.use_curses: self.event_string_len = 0 for evstr in self.events_used.keys(): if self.event_string_len < len(evstr)-3: self.event_string_len = len(evstr)-3 self.pushmany([ "if (physrank == cursestask) {", " /* Indicate which source-code line is currently active. */", "if (thisev->firstline-1 != prevsrcline) {", "mvchgat (prevsrcline, 6, -1, A_NORMAL, 0, NULL);", "(void) touchline (curseswin, prevsrcline, 1);", "if (thisev->firstline-1>=0 && thisev->firstline-1<LINES-1) {", "mvchgat (thisev->firstline-1, 6, -1, A_STANDOUT, 0, NULL);", "(void) touchline (curseswin, thisev->firstline-1, 1);", "}", "prevsrcline = thisev->firstline - 1;", "}", "", " /* Display other useful trace information. */", 'mvprintw (LINES-1, 14+taskdigits, "%*d", taskdigits, thisev->virtrank);', 'mvprintw (LINES-1, 24+2*taskdigits, "%%-%ds", eventnames[thisev->type]);' % self.event_string_len, 'mvprintw (LINES-1, %d+2*taskdigits, "%%*" NICS, eventdigits, i+1);' % (33+self.event_string_len), "", " /* Update the screen and process keyboard commands. */", "(void) refresh ();", "if ((i==0 && breakpoint==0) || ((int)breakpoint==thisev->firstline)) {", " /* Enable single-stepping mode. */", "(void) nocbreak();", "(void) cbreak();", "(void) nodelay (curseswin, FALSE);", "}", "switch (getch()) {", "case 's':", "case 'S':", " /* Enable single-stepping mode. */", "(void) nocbreak();", "(void) cbreak();", "(void) nodelay (curseswin, FALSE);", "break;", "", "case ' ':", " /* Enable normal execution mode. */", "if (cursesdelay)", "(void) halfdelay ((int) ((cursesdelay + 99) / 100));", "else", "(void) nodelay (curseswin, TRUE);", "break;", "", "case 'd':", "case 'D':", " /* Delete the break point. */", "breakpoint = -1;", "break;", "", "case 'q':", "case 'Q':", " /* Quit the program. */", 'ncptl_fatal ("User interactively entered \\"Q\\" to quit the program");', "break;", "", "default:", " /* No other keys do anything special. */", "break;", "}", "}"], stack=newcode) else: self.pushmany([ 'fprintf (stderr, "[TRACE] phys: %d | virt: %d | action: %s | event: %" NICS " / %" NICS " | lines: %d - %d\\n",', "physrank, thisev->virtrank, eventnames[thisev->type], i+1, totalevents, thisev->firstline, thisev->lastline);"], stack=newcode) newcode = newcode + self.invoke_hook("code_define_main_PRE_SWITCH", localvars, invoke_on=self.c_trace_parent) return newcode def code_declare_globals_EXTRA(self, localvars): "Declare additional C global variables needed by the c_trace backend." newvars = [] self.code_declare_var(type="int", name="currentline", arraysize="2", comment="Current lines of source code (beginning and ending)", stack=newvars) self.code_declare_var(name="totalevents", comment="Total # of events in the event list", stack=newvars) if self.use_curses: self.code_declare_var(type="WINDOW *", name="curseswin", comment="Window to use for curses-based tracing", stack=newvars) self.code_declare_var(name="cursestask", comment="Task to trace using curses", stack=newvars) self.code_declare_var(name="cursesdelay", comment="Delay in milliseconds after each curses screen update", stack=newvars) self.code_declare_var(name="breakpoint", comment="Source line at which to enter single-stepping mode", stack=newvars) self.code_declare_var(type="int", name="eventdigits", rhs="1", comment="Number of digits in numevents", stack=newvars) self.code_declare_var(type="int", name="taskdigits", rhs="1", comment="Number of digits in var_num_tasks", stack=newvars) # Make all declarations static. static_newvars = [] for var in newvars: static_newvars.append("static " + var) # Provide a hook for including more variables. static_newvars = static_newvars + self.invoke_hook("code_declare_globals_EXTRA", localvars, invoke_on=self.c_trace_parent) return static_newvars def code_def_init_decls_POST(self, localvars): "Declare extra variables needed within conc_initialize()." newdecls = self.invoke_hook("code_def_init_decls_POST", localvars, invoke_on=self.c_trace_parent) if self.use_curses: self.srcloop = self.code_declare_var(type="int", suffix="loop", comment="Loop over source-code lines", stack=newdecls) return newdecls def code_define_functions_INIT_COMM_3(self, localvars): "Generate code to initialize the c_trace backend." initcode = self.invoke_hook("code_define_functions_INIT_COMM_3", localvars, invoke_on=self.c_trace_parent, after=[""]) if self.use_curses: self.pushmany([ "", " /* Initialize curses. */", "if (physrank == cursestask) {", "if (!(curseswin=initscr()))", 'ncptl_fatal ("Unable to initialize the curses library");', "(void) cbreak();", "(void) noecho();", "(void) curs_set (0);", "if (cursesdelay)", "(void) halfdelay ((int)((cursesdelay+99)/100));", "else", "(void) nodelay (curseswin, TRUE);", "for (%s=0; %s<(int)(sizeof(sourcecode)/sizeof(char *))-1; %s++) {" % (self.srcloop, self.srcloop, self.srcloop), "if (%s >= LINES-1)" % self.srcloop, "break;", "(void) attrset (A_BOLD);", '(void) mvprintw (%s, 0, "%%3d. ", %s+1);' % (self.srcloop, self.srcloop), "(void) attrset (A_NORMAL);", '(void) printw ("%%.*s", COLS, sourcecode[%s]);' % self.srcloop, "}", "(void) refresh();", "}"], stack=initcode) return initcode # Completely redefine codegen_c_generic.py's code_allocate_event method. def code_allocate_event(self, event_type, stack=None, declare="CONC_EVENT *thisev ="): "Push the code to allocate an event and keep track of used events." self.push("%s (currentline[0]=%d, currentline[1]=%d, conc_allocate_event (%s));" % (declare, self.current_node.lineno0, self.current_node.lineno1, event_type), stack) self.events_used[event_type] = 1 def code_def_exit_handler_BODY(self, localvars): "Shut down curses if necessary." if self.use_curses: exitcode = ["if (physrank == cursestask)", "(void) endwin();"] else: exitcode = [] exitcode = exitcode + self.invoke_hook("code_def_exit_handler_BODY", localvars, invoke_on=self.c_trace_parent) return exitcode def n_outputs(self, node): "Write a message to standard out, but not if we're using curses." self.current_node = node self.c_trace_parent.n_outputs(self, node) if self.use_curses: self.arbitrary_code[-1] = []
ld_library_path) sys.stderr.write("# [tcsh] setenv LD_LIBRARY_PATH %s\n" % ld_library_path) return oneline = dynlib_out.readline() dynlib_out.close() except: pass ########################################################################### # The program starts here. if __name__ == "__main__": # Prepare to issue uniform error messages. errmsg = NCPTL_Error("ncptl") # Set default values for our command-line parameters. outfilename = "-" backend = None entirefile = None backend_options = [] filter_list = [] execute_compile = 1 execute_link = 1 keep_ints = 0 lenient = 0 be_verbose = 1 # Determine where coNCePTuaL was installed. try:
class NCPTL_CodeGen: thisfile = globals()["__file__"] #---------------------# # Exported functions # # (called from the # # compiler front end) # #---------------------# def __init__(self, options=None): "Initialize the LibSea code generation module." self.errmsg = NCPTL_Error() # Placeholder until generate is called # Process any arguments we were given. self.source_truncate = 100 # Truncate node source code after this many characters for arg in range(0, len(options)): arg_match = re.match(r'--(node-code)=(.*)', options[arg]) if arg_match: argname, argvalue = arg_match.group(1), arg_match.group(2) if argname == "node-code": argvalue = int(argvalue) if argvalue == -1: self.source_truncate = sys.maxint else: self.source_truncate = argvalue for arg in range(0, len(options)): if options[arg] == "--help": self.show_help() sys.exit(0) def show_help(self): "Output a help message." print """\ Usage: libsea_ast [OPTION...] --node-code=<number> Truncate node source code after this many characters [default: 100] Help options: --help Show this help message""" def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None): "Compile an AST into a list of lines of LibSea code." self.filesource = filesource # Input file self.sourcecode = sourcecode # coNCePTuaL source code self.backend_name = "libsea_graph" self.backend_desc = "parse tree in CAIDA's LibSea graph format" self.errmsg = NCPTL_Error(filesource) self.next_global_ID = 0 # Next LibSea ID to assign to a node # Write a LibSea prologue. self.libseacode = [] if self.filesource == "<command line>": inputfile = "the source program" cleanfilename = "stdin_graph" else: inputfile = os.path.abspath(self.filesource) cleanfilename = (re.sub(r'\W', '_', os.path.splitext(os.path.split(inputfile)[1])[0]) + "_graph") self.libseacode.extend([ "#" * 78, "# This file was generated by coNCePTuaL on %s" % time.asctime(time.localtime(time.time())), "# using the %s backend (%s)." % (self.backend_name, self.backend_desc), "# Do not modify this file; modify %s instead." % inputfile, "#" * 78]) if self.sourcecode: self.libseacode.extend([ "#", "# Entire source program", "# ---------------------"]) for oneline in string.split(string.strip(self.sourcecode), "\n"): self.libseacode.append("# %s" % oneline) self.libseacode.extend([ "#", "#" * 78, ""]) # Acquire information about the graph structure. self.assign_node_IDs(ast) nodes = sorted(self.accumulate_nodes(ast)) nodefmtwidth = int(math.ceil(math.log10(len(nodes)))) links = sorted(self.accumulate_edges(ast)) linkfmtwidth = int(math.ceil(math.log10(len(links)))) # Produce LibSea code for the graph metadata. self.libseacode.extend([ "Graph", "{", " ### metadata ###", ' @name="%s";' % os.path.splitext(os.path.basename(filesource))[0], ' @description="Parse tree for %s";' % inputfile, " @numNodes=%d;" % len(nodes), " @numLinks=%d;" % len(links), " @numPaths=0;", " @numPathLinks=0;", ""]) # Produce LibSea code for the graph structural data. self.libseacode.extend([ " ### structural data ###", " @links=["]) for src, dest in links[:-1]: self.libseacode.append(" { %*d; %*d; }," % \ (linkfmtwidth, src, linkfmtwidth, dest)) self.libseacode.append(" { %*d; %*d; }" % \ (linkfmtwidth, links[-1][0], linkfmtwidth, links[-1][1])) self.libseacode.extend([ " ];", " @paths=;", ""]) # Produce LibSea code for the graph attribute data. self.libseacode.extend([ " ### attribute data ###", " @enumerations=;", " @attributeDefinitions=["]) self.libseacode.extend(self.format_attribute("Type", 1, nodes)) self.libseacode.extend(self.format_attribute("Attribute", 2, nodes)) self.libseacode.extend(self.format_attribute("Source_code", 3, nodes)) self.libseacode.extend(self.format_selection_attr("Is_simple_stmt", [n[0] for n in nodes if n[1] == "simple_stmt"])) self.libseacode.extend(self.format_selection_attr("Is_constant", [n[0] for n in nodes if n[4]])) self.libseacode.extend(self.format_selection_attr("Is_definition", [n[0] for n in nodes if n[5]])) self.libseacode.extend(self.format_selection_attr("Is_leaf", self.accumulate_leaves(ast))) self.libseacode.extend([ " {", " @name=$Is_root_node;", " @type=bool;", " @default=|| false ||;", " @nodeValues=[ { 0; T; } ]; # Root node", " @linkValues=["]) for linknum in range(len(links)-1): self.libseacode.append(" { %*d; T; }," % (linkfmtwidth, linknum)) self.libseacode.extend([ " { %*d; T; }" % (linkfmtwidth, len(links)-1), " ];", " @pathValues=;", " }", " ];", " @qualifiers=[", " {", " @type=$spanning_tree;", " @name=$Parse_tree;", ' @description="Abstract syntax tree corresponding to %s";' % inputfile, " @attributes=[", " { @attribute=7; @alias=$root; },", " { @attribute=7; @alias=$tree_link; }", " ];", " }", " ];", ""]) # Produce LibSea code for the remaining (unused) graph features. self.libseacode.extend([ " ### visualization hints ###", " ; ; ; ;", "", " ### interface hints ###", " ; ; ; ; ;", "}"]) # Return the complete LibSea graph. return self.libseacode def compile_only(self, progfilename, codelines, outfilename, verbose=0, keepints=0): "Output LibSea code." if progfilename == "<command line>": progfilename = "a.out.ncptl" if outfilename == "-": outfilename, _ = os.path.splitext(progfilename) outfilename = outfilename + ".graph" try: outfile = open(outfilename, "w") for oneline in codelines: outfile.write("%s\n" % oneline) outfile.close() except IOError, (errno, strerror): self.errmsg.error_fatal("Unable to produce %s (%s)" % (outfilename, strerror), filename=self.backend_name) if verbose: sys.stderr.write("# Files generated: %s\n" % outfilename)
class NCPTL_CodeGen: def __init__(self, options): "Initialize the profiling module." self.backend_name = "c_profile" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" for arg in range(0, len(options)): profile_match = re.match(r'--profile=(.*)', options[arg]) if profile_match: target_backend = profile_match.group(1) elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--profile=<string>", "Specify a backend to profile") ]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal( "a target backend must be specified using --profile") # Reparent ourselves to the profiled backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_profile"].__bases__ = (immediate_ancestor, ) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor, ) self.name2class["c_profile"] = self.__class__ self.c_profile_parent = self.name2class["c_profile"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.define_eventnames = 1 self.backend_name = "c_profile + " + self.backend_name self.backend_desc = "event profiler atop " + self.backend_desc # ---------------------- # # (Re)implementation of # # hook and other methods # # ---------------------- # def code_declare_globals_EXTRA(self, localvars): "Declare a few arrays to store profile data." newcode = self.invoke_hook("code_declare_globals_EXTRA", localvars, invoke_on=self.c_profile_parent) self.code_declare_var(name="profeventtimings", arraysize="EV_CODE+1", type="static ncptl_int", comment="Total time spent in each event", stack=newcode) self.code_declare_var( name="profeventtallies", arraysize="EV_CODE+1", type="static ncptl_int", comment="Number of times each event was executed", stack=newcode) return newcode def code_define_main_POST_INIT(self, localvars): "Initialize the profile data." newcode = self.invoke_hook("code_define_main_POST_INIT", localvars, invoke_on=self.c_profile_parent) self.push( "memset ((void *)profeventtimings, 0, sizeof(ncptl_int)*(EV_CODE+1));", newcode) self.push( "memset ((void *)profeventtallies, 0, sizeof(ncptl_int)*(EV_CODE+1));", newcode) return newcode def code_def_procev_EVENTS_DECL(self, localvars): "Declare a variable for storing an event starting time." newcode = self.invoke_hook("code_def_procev_EVENTS_DECL", localvars, invoke_on=self.c_profile_parent) self.code_declare_var( name="eventtype", rhs="thisev->type", comment= "Preserved copy of thisev->type in case EV_REPEAT alters thisev", stack=newcode) self.code_declare_var( name="eventstarttime", rhs="ncptl_time()", comment="Time at which the current event began executing", stack=newcode) return newcode def code_def_procev_POST_SWITCH(self, localvars): "Accumulate the time taken by the current event." newcode = self.invoke_hook("code_def_procev_POST_SWITCH", localvars, invoke_on=self.c_profile_parent) self.push( "profeventtimings[eventtype] += ncptl_time() - eventstarttime;", stack=newcode) self.push("profeventtallies[eventtype]++;", stack=newcode) return newcode def code_def_finalize_DECL(self, localvars): "Allocate variables needed to write the profiling information." newcode = self.invoke_hook("code_def_finalize_DECL", localvars, invoke_on=self.c_profile_parent) if self.program_uses_log_file: self.code_declare_var(type="char", name="profilekey", arraysize="256", comment="Space to hold an event name", stack=newcode) self.code_declare_var( type="char", name="profilevalue", arraysize="256", comment="Space to hold a line of profile information", stack=newcode) self.profloopvar = self.code_declare_var( type="int", suffix="ev", comment="Loop over event types", stack=newcode) self.code_declare_var(name="numevents", rhs="ncptl_queue_length (eventqueue)", comment="Total number of events processed", stack=newcode) return newcode def code_def_finalize_PRE(self, localvars): "Write profiling information to either stderr or a log file." newcode = self.invoke_hook("code_def_finalize_PRE", localvars, invoke_on=self.c_profile_parent) profloopvar = self.profloopvar if self.program_uses_log_file: # Write to the log file. self.pushmany([ "for (%s=0; %s<NUM_EVS; %s++)" % ((profloopvar, ) * 3), "if (profeventtallies[%s]) {" % profloopvar, 'sprintf (profilekey, "Profile of %%s (microseconds, count, average)", eventnames[%s]);' % profloopvar, 'sprintf (profilevalue, "%%" NICS " %%" NICS " %%.1f", profeventtimings[%s], profeventtallies[%s], (double)profeventtimings[%s]/(double)profeventtallies[%s]);' % ((profloopvar, ) * 4), "ncptl_log_add_comment (profilekey, profilevalue);", "}", 'strcpy (profilekey, "Profile of event memory");', 'sprintf (profilevalue, "%" NICS " bytes (%" NICS " events * %" NICS " bytes/event)",' "numevents*sizeof(CONC_EVENT), numevents, (ncptl_int)sizeof(CONC_EVENT));", "ncptl_log_add_comment (profilekey, profilevalue);" ], stack=newcode) else: # Write to the standard error device. self.pushmany([ "for (%s=0; %s<NUM_EVS; %s++)" % ((profloopvar, ) * 3), "if (profeventtallies[%s]) {" % profloopvar, 'fprintf (stderr, "%%d %%s %%" NICS " %%" NICS " %%.1f\\n", physrank, eventnames[%s], profeventtimings[%s], profeventtallies[%s], (double)profeventtimings[%s]/(double)profeventtallies[%s]);' % (profloopvar, profloopvar, profloopvar, profloopvar, profloopvar), "}", 'fprintf (stderr, "%d event-memory %" NICS " %" NICS " %" NICS "\n",', "physrank, numevents*sizeof(CONC_EVENT), numevents, (ncptl_int)sizeof(CONC_EVENT));" ], stack=newcode) return newcode
class NCPTL_CodeGen: def __init__(self, options): "Initialize the profiling module." self.backend_name = "c_profile" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" for arg in range(0, len(options)): profile_match = re.match(r'--profile=(.*)', options[arg]) if profile_match: target_backend = profile_match.group(1) elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--profile=<string>", "Specify a backend to profile")]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal("a target backend must be specified using --profile") # Reparent ourselves to the profiled backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_profile"].__bases__ = (immediate_ancestor,) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor,) self.name2class["c_profile"] = self.__class__ self.c_profile_parent = self.name2class["c_profile"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.define_eventnames = 1 self.backend_name = "c_profile + " + self.backend_name self.backend_desc = "event profiler atop " + self.backend_desc # ---------------------- # # (Re)implementation of # # hook and other methods # # ---------------------- # def code_declare_globals_EXTRA(self, localvars): "Declare a few arrays to store profile data." newcode = self.invoke_hook("code_declare_globals_EXTRA", localvars, invoke_on=self.c_profile_parent) self.code_declare_var(name="profeventtimings", arraysize="EV_CODE+1", type="static ncptl_int", comment="Total time spent in each event", stack=newcode) self.code_declare_var(name="profeventtallies", arraysize="EV_CODE+1", type="static ncptl_int", comment="Number of times each event was executed", stack=newcode) return newcode def code_define_main_POST_INIT(self, localvars): "Initialize the profile data." newcode = self.invoke_hook("code_define_main_POST_INIT", localvars, invoke_on=self.c_profile_parent) self.push("memset ((void *)profeventtimings, 0, sizeof(ncptl_int)*(EV_CODE+1));", newcode); self.push("memset ((void *)profeventtallies, 0, sizeof(ncptl_int)*(EV_CODE+1));", newcode); return newcode def code_def_procev_EVENTS_DECL(self, localvars): "Declare a variable for storing an event starting time." newcode = self.invoke_hook("code_def_procev_EVENTS_DECL", localvars, invoke_on=self.c_profile_parent) self.code_declare_var(name="eventtype", rhs="thisev->type", comment="Preserved copy of thisev->type in case EV_REPEAT alters thisev", stack=newcode) self.code_declare_var(name="eventstarttime", rhs="ncptl_time()", comment="Time at which the current event began executing", stack=newcode) return newcode def code_def_procev_POST_SWITCH(self, localvars): "Accumulate the time taken by the current event." newcode = self.invoke_hook("code_def_procev_POST_SWITCH", localvars, invoke_on=self.c_profile_parent) self.push("profeventtimings[eventtype] += ncptl_time() - eventstarttime;", stack=newcode) self.push("profeventtallies[eventtype]++;", stack=newcode) return newcode def code_def_finalize_DECL(self, localvars): "Allocate variables needed to write the profiling information." newcode = self.invoke_hook("code_def_finalize_DECL", localvars, invoke_on=self.c_profile_parent) if self.program_uses_log_file: self.code_declare_var(type="char", name="profilekey", arraysize="256", comment="Space to hold an event name", stack=newcode) self.code_declare_var(type="char", name="profilevalue", arraysize="256", comment="Space to hold a line of profile information", stack=newcode) self.profloopvar = self.code_declare_var(type="int", suffix="ev", comment="Loop over event types", stack=newcode) self.code_declare_var(name="numevents", rhs="ncptl_queue_length (eventqueue)", comment="Total number of events processed", stack=newcode) return newcode def code_def_finalize_PRE(self, localvars): "Write profiling information to either stderr or a log file." newcode = self.invoke_hook("code_def_finalize_PRE", localvars, invoke_on=self.c_profile_parent) profloopvar = self.profloopvar if self.program_uses_log_file: # Write to the log file. self.pushmany([ "for (%s=0; %s<NUM_EVS; %s++)" % ((profloopvar,) * 3), "if (profeventtallies[%s]) {" % profloopvar, 'sprintf (profilekey, "Profile of %%s (microseconds, count, average)", eventnames[%s]);' % profloopvar, 'sprintf (profilevalue, "%%" NICS " %%" NICS " %%.1f", profeventtimings[%s], profeventtallies[%s], (double)profeventtimings[%s]/(double)profeventtallies[%s]);' % ((profloopvar,) * 4), "ncptl_log_add_comment (profilekey, profilevalue);", "}", 'strcpy (profilekey, "Profile of event memory");', 'sprintf (profilevalue, "%" NICS " bytes (%" NICS " events * %" NICS " bytes/event)",' "numevents*sizeof(CONC_EVENT), numevents, (ncptl_int)sizeof(CONC_EVENT));", "ncptl_log_add_comment (profilekey, profilevalue);"], stack=newcode) else: # Write to the standard error device. self.pushmany([ "for (%s=0; %s<NUM_EVS; %s++)" % ((profloopvar,) * 3), "if (profeventtallies[%s]) {" % profloopvar, 'fprintf (stderr, "%%d %%s %%" NICS " %%" NICS " %%.1f\\n", physrank, eventnames[%s], profeventtimings[%s], profeventtallies[%s], (double)profeventtimings[%s]/(double)profeventtallies[%s]);' % (profloopvar, profloopvar, profloopvar, profloopvar, profloopvar), "}", 'fprintf (stderr, "%d event-memory %" NICS " %" NICS " %" NICS "\n",', "physrank, numevents*sizeof(CONC_EVENT), numevents, (ncptl_int)sizeof(CONC_EVENT));"], stack=newcode) return newcode
class NCPTL_Lexer: def __init__(self): "Initialize the lexer." # Define a mapping from each uppercase keyword to its # canonicalized form. self.canonicalize_kw = { "A" : "AN", "AWAIT" : "AWAITS", "BIT" : "BITS", "BUFFER" : "BUFFERS", "BYTE" : "BYTES", "COMPLETION" : "COMPLETIONS", "COMPUTE" : "COMPUTES", "DAY" : "DAYS", "DOUBLEWORD" : "DOUBLEWORDS", "EXECUTE" : "EXECUTES", "HALFWORD" : "HALFWORDS", "HOUR" : "HOURS", "INTEGER" : "INTEGERS", "IS" : "ARE", "IT" : "THEM", "ITS" : "THEIR", "LOG" : "LOGS", "MESSAGE" : "MESSAGES", "MICROSECOND" : "MICROSECONDS", "MILLISECOND" : "MILLISECONDS", "MINUTE" : "MINUTES", "MULTICAST" : "MULTICASTS", "OUTPUT" : "OUTPUTS", "PAGE" : "PAGES", "PROCESSOR" : "PROCESSORS", "QUADWORD" : "QUADWORDS", "RECEIVE" : "RECEIVES", "REPETITION" : "REPETITIONS", "REDUCE" : "REDUCES", "RESET" : "RESETS", "RESTORE" : "RESTORES", "RESULT" : "RESULTS", "SECOND" : "SECONDS", "SEND" : "SENDS", "SLEEP" : "SLEEPS", "STORE" : "STORES", "SYNCHRONIZE" : "SYNCHRONIZES", "TASK" : "TASKS", "TIME" : "TIMES", "TOUCH" : "TOUCHES", "WORD" : "WORDS"} for kw in map(string.upper, Keywords.keywords): self.canonicalize_kw[kw] = self.canonicalize_kw.get(kw, kw) # Define a list of token names. tokens = {} for ckw in self.canonicalize_kw.values(): tokens[ckw] = 1 tokens = tokens.keys() tokens.extend(["comma", "ellipsis", "ident_token", "integer", "lbrace", "lbracket", "logic_and", "logic_or", "lparen", "op_and", "op_div", "op_eq", "op_geq", "op_gt", "op_leq", "op_lshift", "op_lt", "op_minus", "op_mult", "op_neq", "op_or", "op_plus", "op_power", "op_rshift", "period", "rbrace", "rbracket", "rparen", "star", "string_token"]) self.tokens = tokens def tokenize(self, sourcecode, filesource='<stdin>'): "Tokenize the given string of source code." self.errmsg = NCPTL_Error(filesource) # Keep track of all the comments we've encountered by storing # a mapping from line number to comment (including the initial # hash character). self.line2comment = {} # Initialize the lexer. lex.lex(module=self) # Repeatedly invoke the lexer and return all of the tokens it produces. self.lineno = 1 lex.input(sourcecode) self.toklist = [] while 1: # Acquire the next token and assign it a line number if necessary. token = lex.token() if not token: break if token.lineno < self.lineno: token.lineno = self.lineno # Hack: Disambiguate op_mult and star on the parser's behalf. if token.type in ["comma", "rparen"]: try: if self.toklist[-1].type == "op_mult": self.toklist[-1].type = "star" except IndexError: pass # We now have one more valid token. self.toklist.append(token) return self.toklist # Define a bunch of simple token types. t_comma = r' , ' t_ellipsis = r' \.\.\. ' t_lbrace = r' \{ ' t_lbracket = r' \[ ' t_logic_and = r' /\\ ' t_logic_or = r' \\/ ' t_lparen = r' \( ' t_op_and = r' & ' t_op_div = r' / ' t_op_eq = r' = ' t_op_geq = r' >= ' t_op_gt = r' > ' t_op_leq = r' <= ' t_op_lshift = r' << ' t_op_lt = r' < ' t_op_minus = r' - ' t_op_mult = r' \* ' t_op_neq = r' <> ' t_op_or = r' \| ' t_op_plus = r' \+ ' t_op_power = r' \*\* ' t_op_rshift = r' >> ' t_period = r' \. ' t_rbrace = r' \} ' t_rbracket = r' \] ' t_rparen = r' \) ' # Keep track of line numbers. def t_newline(self, token): r' \r?\n ' self.lineno = self.lineno + 1 return None # Ignore whitespace. def t_whitespace(self, token): r' [ \t]+ ' return None # Remove comments. def t_comment(self, token): r' \#.* ' self.line2comment[self.lineno] = token.value return None # Sanitize and store string literals. def t_string_token(self, token): r' \"([^\\]|(\\[\000-\177]))*?\" ' sanitized = [] c = 1 while c < len(token.value)-1: onechar = token.value[c] if onechar == "\\": c = c + 1 onechar = token.value[c] if onechar == "n": sanitized.append("\n") elif onechar == "t": sanitized.append("\t") elif onechar == "r": sanitized.append("\r") elif onechar == "\n": self.lineno = self.lineno + 1 elif onechar in ["\\", '"']: sanitized.append(onechar) else: self.errmsg.warning('Discarding unrecognized escape sequence "\\%s"' % onechar, lineno0=self.lineno, lineno1=self.lineno) else: sanitized.append(onechar) if onechar == "\n": self.lineno = self.lineno + 1 c = c + 1 token.value = '"%s"' % string.join(sanitized, "") token.lineno = self.lineno return token # Store idents as "ident" and keywords as themselves (uppercased). def t_ident_or_keyword(self, token): r' [A-Za-z]\w* ' try: # Store a keyword with its value (uppercase) as its type. token.type = self.canonicalize_kw[string.upper(token.value)] if len(self.toklist) > 0 and self.toklist[-1].value == "-": # A "-" before a keyword is treated as whitespace. self.toklist.pop() except KeyError: # Store an identifier with a tuple for a value: # {lowercase, original}. token.type = "ident_token" token.value = (string.lower(token.value), token.value) token.lineno = self.lineno return token # Store an integer as a tuple {long-expanded, original}. Note # that coNCePTuaL integers can contain a trailing multiplier, a # trailing exponent, and a trailing "st", "nd", "rd", or "th". def t_integer(self, token): r' \d+([KMGkmg]|([Ee]\d+))?([Ss][Tt]|[NnRr][Dd]|[Tt][Hh]?)? ' canon_token = re.sub(r'(st|nd|rd|th)$', "", string.lower(token.value)) parts = re.split(r'([kmgte])', canon_token, 1) if not parts[-1]: parts = parts[:-1] number = long(parts[0]) if len(parts) == 2: if parts[1] == "k": number = number * 1024L elif parts[1] == "m": number = number * 1024L**2 elif parts[1] == "g": number = number * 1024L**3 elif parts[1] == "t": number = number * 1024L**4 else: self.errmsg.error_syntax(token.value, lineno0=token.lineno, lineno1=token.lineno) elif len(parts) == 3: number = number * 10**long(parts[2]) token.value = (number, token.value) token.lineno = self.lineno return token # Everything else we encounter should return a syntax error. def t_error(self, token): self.errmsg.error_syntax(token.value[0], lineno0=token.lineno, lineno1=token.lineno)
def __init__(self, options): "Initialize the execution-trace module." self.backend_name = "c_trace" self.errmsg = NCPTL_Error(self.backend_name) # Process any arguments we were given. leftover_opts = [] target_backend = "" self.use_curses = 0 for arg in range(0, len(options)): trace_match = re.match(r'--trace=(.*)', options[arg]) if trace_match: target_backend = trace_match.group(1) elif options[arg] == "--curses": self.use_curses = 1 elif options[arg] == "--help": # Utilize c_generic's help-string mechanism. import codegen_c_generic generic_self = codegen_c_generic.NCPTL_CodeGen() generic_self.backend_name = self.backend_name generic_self.cmdline_options.extend([ ("--trace=<string>", "Specify a backend to trace"), ("--curses", """Display the trace with curses instead of with fprintf()""") ]) generic_self.show_help() raise SystemExit, 0 else: leftover_opts.append(options[arg]) if not target_backend: self.errmsg.error_fatal( "a target backend must be specified using --trace") # Reparent ourselves to the traced backend. try: exec("import codegen_%s" % target_backend) exec("immediate_ancestor = codegen_%s.NCPTL_CodeGen" % target_backend) except: self.errmsg.error_fatal("Unable to initialize the %s backend" % target_backend) try: self.name2class[target_backend] = immediate_ancestor except AttributeError: self.name2class = {target_backend: immediate_ancestor} try: self.name2class["c_trace"].__bases__ = (immediate_ancestor, ) except KeyError: # We're the top-level class. self.__class__.__bases__ = (immediate_ancestor, ) self.name2class["c_trace"] = self.__class__ self.c_trace_parent = self.name2class["c_trace"].__bases__[0] immediate_ancestor.__init__(self, leftover_opts) self.intercept_node_funcs(self.name2class["c_trace"]) if self.use_curses: self.set_param("LIBS", "prepend", "-lcurses") self.define_eventnames = 1 self.backend_name = "c_trace + " + self.backend_name self.backend_desc = "event tracer atop " + self.backend_desc # Add a command-line option to specify which task should be monitored. if self.use_curses: self.base_global_parameters.extend([ ("NCPTL_TYPE_INT", "cursestask", "monitor", "M", "Processor to monitor", "0"), ("NCPTL_TYPE_INT", "cursesdelay", "delay", "D", "Delay in milliseconds after each screen update (0=no delay)", "0"), ("NCPTL_TYPE_INT", "breakpoint", "breakpoint", "B", "Source line at which to enter single-stepping mode (-1=none; 0=first event)", "-1") ])
def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None): "Compile an AST into a list of lines of LibSea code." self.filesource = filesource # Input file self.sourcecode = sourcecode # coNCePTuaL source code self.backend_name = "libsea_graph" self.backend_desc = "parse tree in CAIDA's LibSea graph format" self.errmsg = NCPTL_Error(filesource) self.next_global_ID = 0 # Next LibSea ID to assign to a node # Write a LibSea prologue. self.libseacode = [] if self.filesource == "<command line>": inputfile = "the source program" cleanfilename = "stdin_graph" else: inputfile = os.path.abspath(self.filesource) cleanfilename = (re.sub(r'\W', '_', os.path.splitext(os.path.split(inputfile)[1])[0]) + "_graph") self.libseacode.extend([ "#" * 78, "# This file was generated by coNCePTuaL on %s" % time.asctime(time.localtime(time.time())), "# using the %s backend (%s)." % (self.backend_name, self.backend_desc), "# Do not modify this file; modify %s instead." % inputfile, "#" * 78]) if self.sourcecode: self.libseacode.extend([ "#", "# Entire source program", "# ---------------------"]) for oneline in string.split(string.strip(self.sourcecode), "\n"): self.libseacode.append("# %s" % oneline) self.libseacode.extend([ "#", "#" * 78, ""]) # Acquire information about the graph structure. self.assign_node_IDs(ast) nodes = sorted(self.accumulate_nodes(ast)) nodefmtwidth = int(math.ceil(math.log10(len(nodes)))) links = sorted(self.accumulate_edges(ast)) linkfmtwidth = int(math.ceil(math.log10(len(links)))) # Produce LibSea code for the graph metadata. self.libseacode.extend([ "Graph", "{", " ### metadata ###", ' @name="%s";' % os.path.splitext(os.path.basename(filesource))[0], ' @description="Parse tree for %s";' % inputfile, " @numNodes=%d;" % len(nodes), " @numLinks=%d;" % len(links), " @numPaths=0;", " @numPathLinks=0;", ""]) # Produce LibSea code for the graph structural data. self.libseacode.extend([ " ### structural data ###", " @links=["]) for src, dest in links[:-1]: self.libseacode.append(" { %*d; %*d; }," % \ (linkfmtwidth, src, linkfmtwidth, dest)) self.libseacode.append(" { %*d; %*d; }" % \ (linkfmtwidth, links[-1][0], linkfmtwidth, links[-1][1])) self.libseacode.extend([ " ];", " @paths=;", ""]) # Produce LibSea code for the graph attribute data. self.libseacode.extend([ " ### attribute data ###", " @enumerations=;", " @attributeDefinitions=["]) self.libseacode.extend(self.format_attribute("Type", 1, nodes)) self.libseacode.extend(self.format_attribute("Attribute", 2, nodes)) self.libseacode.extend(self.format_attribute("Source_code", 3, nodes)) self.libseacode.extend(self.format_selection_attr("Is_simple_stmt", [n[0] for n in nodes if n[1] == "simple_stmt"])) self.libseacode.extend(self.format_selection_attr("Is_constant", [n[0] for n in nodes if n[4]])) self.libseacode.extend(self.format_selection_attr("Is_definition", [n[0] for n in nodes if n[5]])) self.libseacode.extend(self.format_selection_attr("Is_leaf", self.accumulate_leaves(ast))) self.libseacode.extend([ " {", " @name=$Is_root_node;", " @type=bool;", " @default=|| false ||;", " @nodeValues=[ { 0; T; } ]; # Root node", " @linkValues=["]) for linknum in range(len(links)-1): self.libseacode.append(" { %*d; T; }," % (linkfmtwidth, linknum)) self.libseacode.extend([ " { %*d; T; }" % (linkfmtwidth, len(links)-1), " ];", " @pathValues=;", " }", " ];", " @qualifiers=[", " {", " @type=$spanning_tree;", " @name=$Parse_tree;", ' @description="Abstract syntax tree corresponding to %s";' % inputfile, " @attributes=[", " { @attribute=7; @alias=$root; },", " { @attribute=7; @alias=$tree_link; }", " ];", " }", " ];", ""]) # Produce LibSea code for the remaining (unused) graph features. self.libseacode.extend([ " ### visualization hints ###", " ; ; ; ;", "", " ### interface hints ###", " ; ; ; ; ;", "}"]) # Return the complete LibSea graph. return self.libseacode
except KeyError: ld_library_path = libdir sys.stderr.write("# [bash] export LD_LIBRARY_PATH=%s\n" % ld_library_path) sys.stderr.write("# [tcsh] setenv LD_LIBRARY_PATH %s\n" % ld_library_path) return oneline = dynlib_out.readline() dynlib_out.close() except: pass ########################################################################### # The program starts here. if __name__ == "__main__": # Prepare to issue uniform error messages. errmsg = NCPTL_Error("ncptl") # Set default values for our command-line parameters. outfilename = "-" backend = None entirefile = None backend_options = [] filter_list = [] execute_compile = 1 execute_link = 1 keep_ints = 0 lenient = 0 be_verbose = 1 # Determine where coNCePTuaL was installed. try:
def generate(self, ast, filesource='<stdin>', filetarget="-", sourcecode=None): "Compile an AST into a list of lines of DOT code." self.filesource = filesource # Input file self.sourcecode = sourcecode # coNCePTuaL source code self.backend_name = "dot_ast" self.backend_desc = "parse tree in AT&T's DOT language" self.errmsg = NCPTL_Error(filesource) # Write a DOT prologue. self.dotcode = [] if self.filesource == "<command line>": inputfile = "the source program" cleanfilename = "stdin_graph" else: inputfile = os.path.abspath(self.filesource) cleanfilename = (re.sub(r'\W', '_', os.path.splitext(os.path.split(inputfile)[1])[0]) + "_graph") self.dotcode.extend([ "// " + "*" * 70, "// This file was generated by coNCePTuaL on %s" % time.asctime(time.localtime(time.time())), "// using the %s backend (%s)." % (self.backend_name, self.backend_desc), "// Do not modify this file; modify %s instead." % inputfile, "// " + "*" * 70]) if self.sourcecode: self.dotcode.extend([ "//", "// Entire source program", "// ---------------------"]) for oneline in string.split(string.strip(self.sourcecode), "\n"): self.dotcode.append("// %s" % oneline) self.dotcode.extend([ "", "", "digraph %s {" % cleanfilename, " /* Graph defaults */", ' page = "8.5, 11";', ' size = "7.5, 10";', " node [shape=record];", ""]) if self.extra_dot_code: self.dotcode.append(" /* Extra code specified on the command line */") self.dotcode.extend(map(lambda str: " %s;" % re.sub(r'[\s;]+$', "", str), self.extra_dot_code)) self.dotcode.append("") # Walk the AST in postorder fashion to produce a graphical parse tree. self.dotcode.append(" /* The program-specific parse tree */") self.dotcode.append(" subgraph cluster_parse_tree {") self.dotcode.append(' style = "invis";') self.nextnodenum = 1 if self.compress_graph: self.elide_chains(ast) self.postorder_traversal(ast) self.dotcode.append(" }") self.dotcode.append("") # Optionally output the source program. if self.show_source_code: self.dotcode.append(" /* Program source code */") self.dotcode.append(" subgraph cluster_source_code {") self.dotcode.append(' style = "invis";') lineno = 1 codelines = [] for oneline in string.split(string.rstrip(self.sourcecode), "\n"): codelines.append("%6d) %s" % (lineno, self.string_to_dot(oneline))) lineno = lineno + 1 self.dotcode.append(' source_code [shape=plaintext, fontname="Courier",') self.dotcode.append(' label="%s\\l"];' % string.join(codelines, "\\l")) self.dotcode.append(" }") # Write a DOT footer. self.dotcode.append("}") return self.dotcode