def dedup_exec(outputdir): tag = 'dedup_exec' (dedup_stdout, dedup_stderr) = dedup_init(outputdir) target_pids = [] tracer = traceinfo('dedup') success = tracer.trace_on(outputdir, descr='starting dedup') if success: (success, dedup_pid) = run_dedup(outputdir, dedup_stdout, dedup_stderr, tracer) (tracesuccess, buffer_full) = tracer.trace_off(descr='dedup complete') if success: if not tracesuccess or buffer_full: print_error(tag, ("trace buffer filled up before " "tracing turned off - considering this an error " "here, but echo {} > target_pids file to analyze " "trace anyway").format(dedup_pid)) success = False target_pids = [] else: target_pids.append(dedup_pid) else: print_error(tag, ("trace_on() or run_dedup() returned failure; " "will just clean up and return now. target_pids will be " "empty, but echo {} > target_pids file to analyze " "trace anyway").format(dedup_pid)) dedup_cleanup([dedup_stdout, dedup_stderr]) return target_pids
def browser_exec(outputdir, which): tag = 'browser_exec' if which not in valid_browsers: print_error(tag, ("invalid browser: {}").format(which)) return [] #print_debug(tag, ("os.environ: {}").format(os.environ)) # The output_dir already distinguishes between firefox and chrome. (browser_stdout, browser_stderr) = browser_init(outputdir) target_pids = [] tracer = traceinfo(which) success = tracer.trace_on(outputdir, descr="starting browser {}".format(which)) if success: (retcode, browser_pid) = run_browser(outputdir, browser_stdout, browser_stderr, which, tracer) if retcode == 'success': success = True else: print_error(tag, ("run_browser() returned {}, considering " "this an error here. echo {} > target_pids to " "analyze trace anyway").format(retcode, browser_pid)) success = False (tracesuccess, buffer_full) = tracer.trace_off( descr="{} complete".format(which), targetpid=browser_pid) if retcode == 'full': # checkpoints from inside run_browser() buffer_full = True #print_debug(tag, ("os.environ: {}").format(os.environ)) if success: if not tracesuccess: print_error(tag, ("tracesuccess is false, returning " "error").format()) success = False target_pids = [] elif buffer_full: print_error(tag, ("trace buffer filled up; still considered " "an error for now, but if you want to, echo {} > " "target_pids file to analyze trace anyway").format( browser_pid)) success = False target_pids = [] else: print_debug(tag, ("").format(buffer_full)) target_pids.append(browser_pid) else: print_error(tag, ("trace_on() or run_browser() returned failure; " "will just clean up and return now. target_pids will be " "empty, but echo {} > target_pids file to analyze " "trace anyway").format(browser_pid)) browser_cleanup([browser_stdout, browser_stderr]) return target_pids
def hw_exec(outputdir, static_dynamic): tag = 'hw_exec' if static_dynamic == 'dynamic': name = 'helloworld' elif static_dynamic == 'static': name = 'helloworld-static' else: print_error(tag, ("invalid arg: static_dynamic={}").format( static_dynamic)) return [] (hw_stdout, hw_stderr) = hw_init(outputdir) tracer = traceinfo(name) target_pids = [] success = tracer.trace_on(outputdir, "starting {}".format(name), use_perf=True) if not success: print_error(tag, ("trace_on failed, returning [] now").format()) hw_cleanup([hw_stdout, hw_stderr]) return [] (success, hw_pid) = run_hw(static_dynamic, outputdir, hw_stdout, hw_stderr, tracer) if success and hw_pid > 1: target_pids.append(hw_pid) print_debug(tag, ("run_hw() successful, target_pids: " "{}").format(target_pids)) else: print_error(tag, ("run_hw() returned {} and {}; will " "return empty target_pids list").format(success, hw_pid)) (tracesuccess, buffer_full) = tracer.trace_off( descr="{} done".format(name)) if not tracesuccess or buffer_full: print_error(tag, ("trace buffer filled up before " "tracing turned off - considering this an error " "here").format()) success = False target_pids = [] hw_cleanup([hw_stdout, hw_stderr]) return target_pids
def office_exec(outputdir): tag = 'office_exec' (office_stdout, office_stderr) = new_out_err_files(outputdir, 'office') target_pids = [] tracer = traceinfo('office') # Start+stop the X virtual frame buffer outside of the trace. if headless: (xvfb_p, display) = start_xvfb(outputdir) if xvfb_p is None or display is None: print_error(tag, ("start_xvfb returned error, returning " "[] without starting office")) return [] else: (xvfb_p, display) = (None, None) success = tracer.trace_on(outputdir, descr="starting LibreOffice".format()) if success: (success, office_pid) = run_office(outputdir, office_stdout, office_stderr, display, tracer) (tracesuccess, buffer_full) = tracer.trace_off( descr="LibreOffice complete".format()) if success: if not tracesuccess or buffer_full: print_error(tag, ("trace buffer filled up before " "tracing turned off - considering this an error " "here").format()) success = False target_pids = [] else: target_pids.append(office_pid) else: print_error(tag, ("trace_on() or run_office() returned failure; " "will just clean up and return now. target_pids will be " "empty...").format()) if headless: stop_xvfb(xvfb_p) office_cleanup([office_stdout, office_stderr]) return target_pids
def py_exec(outputdir): tag = "py_exec" target_pids = [] (py_stdout, py_stderr) = py_init(outputdir) tracer = traceinfo("python") success = tracer.trace_on(outputdir, "starting python") if not success: print_error(tag, ("trace_on failed, returning [] now").format()) py_cleanup([py_stdout, py_stderr]) return [] (success, py_pid) = run_py_script(outputdir, py_stdout, py_stderr, tracer) if success and py_pid > 1: target_pids.append(py_pid) print_debug(tag, ("run_py_script() successful, target_pids: " "{}").format(target_pids)) else: print_error( tag, ("run_py_script() returned {} and {}; will " "return empty target_pids list").format(success, py_pid) ) (tracesuccess, buffer_full) = tracer.trace_off(descr="python done".format()) if not tracesuccess or buffer_full: print_error( tag, ( "trace buffer filled up before " "tracing turned off - considering this an error " "here, but echo {} > target_pids file to analyze " "trace anyway" ).format(py_pid), ) success = False target_pids = [] py_cleanup([py_stdout, py_stderr]) return target_pids
def cassandra_exec(output_dir): tag = 'cassandra_exec' (success, cass_stdout, cass_stderr, ycsb_stdout, ycsb_stderr) = cass_init(output_dir) if not success: print_error(tag, ("cass_init failed, returning no target_pids")) return [] target_pids = [] tracer = traceinfo('cassandra') # If any of the trace steps fails along the way, we can skip some # of the rest of the steps, but make sure to still stop the cassandra # server and turn tracing off. success = True # cass_init will start and then stop the Cassandra server, because # we want to trace Cassandra while it starts from scratch: success = tracer.trace_on(output_dir, descr='starting Cassandra trace') if not success: print_error(tag, ("trace_on failed, returning before starting " "cassandra or ycsb").format()) return [] cass_p = start_cass_server(output_dir, cass_stdout, cass_stderr) retcode = tracer.trace_checkpoint('after-cass-start', targetpid=cass_p.pid) if retcode == 'error' or retcode == 'full': print_error(tag, ("trace_checkpoint returned {} after " "cassandra start, this is unexpected - will return empty " "target_pids list").format(retcode)) success = False if success: ycsb_params_fname = ycsb_load(output_dir, ycsb_stdout, ycsb_stderr) if not ycsb_params_fname: print_error(tag, ("ycsb_load failed")) success = False retcode = tracer.trace_checkpoint('after-ycsb-load', targetpid=cass_p.pid) if retcode == 'error' or retcode == 'full': print_error(tag, ("trace_checkpoint returned {} after " "ycsb load, this is unexpected - will return empty " "target_pids list").format(retcode)) success = False if success: if not tracer.perf_on(): print_error(tag, ("perf_on() failed, but continuing")) success = ycsb_run(ycsb_params_fname, output_dir, ycsb_stdout, ycsb_stderr) tracer.perf_off() retcode = tracer.trace_checkpoint('after-ycsb-run', targetpid=cass_p.pid) if retcode == 'error' or retcode == 'full': print_error(tag, ("trace_checkpoint returned {} after " "ycsb run, this is unexpected - will return empty " "target_pids list").format(retcode)) success = False stop_cass_server(cass_p) if success: retcode = tracer.trace_checkpoint('after-cass-shutdown') if retcode == 'error' or retcode == 'full': print_error(tag, ("trace_checkpoint returned {} after " "cassandra shutdown, this is unexpected - will return empty " "target_pids list").format(retcode)) success = False (tracesuccess, buffer_full) = tracer.trace_off( descr='ending Cassandra trace') cass_cleanup([cass_stdout, cass_stderr, ycsb_stdout, ycsb_stderr]) if success: if not tracesuccess or buffer_full: print_error(tag, ("trace buffer filled up before " "tracing turned off - considering this an error " "here").format()) success = False target_pids = [] else: target_pids.append(cass_p.pid) return target_pids
def manual_loop(output_dir, manual_stdout, manual_stderr): tag = 'manual_loop' # Options for reading input from shell: # http://docs.python.org/3/library/functions.html#input # http://docs.python.org/3/library/cmd.html print(("NOTE: you should add at least one \"target pid\" for the\n" "app(s) you run manually, otherwise later steps may fail!").format()) prompt = ("'t' = toggle tracing; '<name>' = take named checkpoint; " "<Enter> = take checkpoint; 'p' = add target pid; " "Ctrl-d = quit\n--> ").format() tracer = traceinfo('manualapp') target_pids = [] success = True trace_active = False cpnum = 1 while(True): try: cmd = input(prompt) if cmd == 't': if not trace_active: success = tracer.trace_on(output_dir, descr='manual toggle on') if not success: print_error(tag, ("trace_on failed!").format()) break trace_active = True if not tracer.perf_on(): print_error(tag, ("perf_on() failed, but continuing")) print(("\ttoggled tracing ON; run your app in another " "shell!")) else: tracer.perf_off() (success, buffer_full) = tracer.trace_off( descr='manual toggle off') trace_active = False if buffer_full: print("\tWARNING: trace buffer filled before " "trace turned off!") if not success: break print("\ttoggled tracing OFF") elif cmd == 'p': pidprompt = ("Enter target pid> ").format() pidstr = input(pidprompt) try: pid = int(pidstr) target_pids.append(pid) print(("\ttarget_pids list: {}").format(target_pids)) except ValueError: print(("\tinvalid pid input \"{}\"").format( pidstr)) else: if trace_active: if cmd == '': cpname = "checkpoint{}".format( str(cpnum).zfill(3)) cpnum += 1 else: cpname = cmd retcode = tracer.trace_checkpoint(cpname) if retcode == 'full': print("\ttrace buffer is full! No checkpoint taken") elif retcode == 'error': print("\terror while taking checkpoint") success = False break print("\ttook checkpoint: {}".format(cpname)); else: print("\tcan't take checkpoint, tracing is off"); except EOFError: print('') # newline after prompt break if trace_active: (success, buffer_full) = tracer.trace_off( descr='final manual toggle off') trace_active = False if not success or buffer_full: print("\tWARNING: trace buffer filled before trace " "turned off!") print("\ttoggled tracing OFF") if not success: print_error(tag, ("trace_off may have failed, make sure that " "kernel tracing is not still active!").format()) return target_pids
def kernelbuild_exec(outputdir): tag = 'kernelbuild_exec' # Easiest to just cd to the linux source directory for the duration # of this script, then cd back when we're done. prev_cwd = set_cwd(LINUXSRCDIR) if not prev_cwd: print_error(tag, ("set_cwd failed, returning now")) return success = True tracer = traceinfo('kernelbuild') if BUILDTYPE == 'null': print_debug(tag, ("BUILDTYPE {}: building once without tracing " "so that next build will be a 'null' build").format(BUILDTYPE)) (build_stdout, build_stderr) = build_init(outputdir, 'prelim') success = run_kernelbuild(outputdir, build_stdout, build_stderr, BUILDTYPE, tracer) build_cleanup([build_stdout, build_stderr]) elif BUILDTYPE == 'full': print_debug(tag, ("BUILDTYPE {}: building clean without tracing, " "so that next build will be a full build").format(BUILDTYPE)) (build_stdout, build_stderr) = build_init(outputdir, 'clean') success = run_kernelbuild(outputdir, build_stdout, build_stderr, BUILDTYPE, tracer) build_cleanup([build_stdout, build_stderr]) elif BUILDTYPE == 'onefile': print_debug(tag, ("BUILDTYPE {}: calling touch_kernelfile(), " "then will perform traced build").format(BUILDTYPE)) success = touch_kernelfile() elif BUILDTYPE == 'default': print_debug(tag, ("BUILDTYPE {}: building with tracing " "immediately, using whatever state source dir was in").format( BUILDTYPE)) else: print_error(tag, ("invalid BUILDTYPE {}, returning now").format( BUILDTYPE)) unset_cwd(prev_cwd) return if not success: print_error(tag, ("initial {} operation failed, returning " "now").format(BUILDTYPE)) unset_cwd(prev_cwd) return target_pids = [] (build_stdout, build_stderr) = build_init(outputdir, 'traced') success = tracer.trace_on(outputdir, descr='starting kernelbuild', use_perf=True) if success: (success, build_pid) = run_kernelbuild(outputdir, build_stdout, build_stderr, 'traced', tracer) (tracesuccess, buffer_full) = tracer.trace_off( descr='kernelbuild complete') if success: if not tracesuccess: print_error(tag, ("tracesuccess is False, this is an " "error; returning empty target_pids (without " "top-level make pid {}").format(build_pid)) success = False target_pids = [] elif buffer_full: print_warning(tag, ("trace buffer filled up before " "tracing turned off, but won't consider this an " "error here; returning success")) success = True target_pids.append(build_pid) else: target_pids.append(build_pid) print_debug(tag, ("target_pids: {}").format(target_pids)) else: print_error(tag, ("trace_on() or run_kernelbuild() returned " "failure; will just cleanup and return now. target_pids " "will be empty...").format()) build_cleanup([build_stdout, build_stderr]) now_cwd = unset_cwd(prev_cwd) if not now_cwd: print_error(tag, ("failed to cd back to {}").format(prev_cwd)) print_debug(tag, ("returning target_pids: {}").format(target_pids)) return target_pids
def mediawiki_exec(outputdir, services, manualservice=None): tag = 'mediawiki_exec' stdouts = [] stderrs = [] target_pids = [] service_pids = [] target_service = services[-1] tracer = traceinfo(target_service) tracesuccess = False # Initialization: set up stdout and stderr files, and stop # any services that may be running. for service in services: (service_stdout, service_stderr) = service_init(outputdir, service) stdouts.append(service_stdout) stderrs.append(service_stderr) (success, service_pid) = ubuntu_services.service_cmd( service, 'stop', outputdir, service_stdout, service_stderr) if USE_MANUAL_CLIENT: print_debug(tag, ("skipping browser client start")) (client, xvfb_p) = (None, None) else: # start_client() returns a client driver and the subprocess.Popen # object for the virtual frame buffer (if we're in headless mode). (client, xvfb_p) = start_client(outputdir) if not client: service_cleanup(stdouts + stderrs) print_error(tag, ("start_client() failed, returning empty " "target_pids")) return [] print_debug(tag, ("client started successfully").format()) # Start all services, enabling tracing before starting the last # one: for i in range(len(services)): service = services[i] if i == len(services) - 1 and manualservice is None: success = tracer.trace_on(outputdir, descr="starting {}".format(service)) if not success: print_error(tag, ("trace_on failed, breaking out of " "start loop")) break (success, service_pid) = ubuntu_services.service_cmd( service, 'start', outputdir, stdouts[i], stderrs[i]) if success and service_pid > 1: service_pids.append(service_pid) print_debug(tag, ("appended service_pid {} for {}").format( service_pids[-1], service)) else: print_error(tag, ("start {} returned {} or could not find " "pid ({}), so breaking out of start loop").format( service, success, service_pid)) break tracemanualservice = False if success and manualservice != None: if tracemanualservice: success = tracer.trace_on(outputdir, descr="starting {}".format(service)) if success: signal.signal(signal.SIGINT, signal_handler_nop) print(("Ok, tracing is on, start the {} service, wait " "a little while (perhaps issue a separate web request " "to verbena.cs.washington.edu/mediawiki/index.php/" "Navidad to ensure mediawiki setup is " "working) and " "press Ctrl-C to begin the client workload. While " "workload is running, note the pid of your " "service!").format(manualservice)) signal.pause() # Note: Linux-only signal.signal(signal.SIGINT, signal.SIG_DFL) success = True if success: if USE_MANUAL_CLIENT: # Pause until Ctrl-C (SIGINT) is received. Call a nop signal # handler when signal is received, then reset signal behavior # back to default. # http://docs.python.org/3/library/signal.html signal.signal(signal.SIGINT, signal_handler_nop) print(("Paused - execute your client workload then press " "Ctrl-C to stop trace.")) signal.pause() # Note: Linux-only signal.signal(signal.SIGINT, signal.SIG_DFL) success = True else: success = run_client(client, outputdir, tracer, service_pids[-1]) if success and manualservice is None: target_pids.append(service_pids[-1]) print_debug(tag, ("client succeeded, {} target_pids: " "{}").format(target_service, target_pids)) elif success and manualservice != None: signal.signal(signal.SIGINT, signal_handler_nop) print(("Ok, client workload completed, kill the {} service " "then hit Ctrl-C to turn off tracing and stop other " "services.").format(manualservice)) signal.pause() # Note: Linux-only signal.signal(signal.SIGINT, signal.SIG_DFL) success = True else: if len(service_pids) > 0: tpid = service_pids[-1] else: tpid = -1234 print_error(tag, ("service-start or trace_on or run_client " "returned failure; will just clean up and return now. " "target_pids will be empty, but echo {} > target_pids " "to analyze trace anyway").format(tpid)) for i in range(len(services) - 1, -1, -1): service = services[i] (success, ignore) = ubuntu_services.service_cmd( service, 'stop', outputdir, stdouts[i], stderrs[i]) if not success: print_error(tag, ("received error on {} stop, will " "continue stopping other services").format(service)) # Stop trace *after* stopping target service: if (i == len(services) - 1 and (manualservice is None or tracemanualservice)): (tracesuccess, buffer_full) = tracer.trace_off( descr="stopping {}".format(service), targetpid=service_pids[i]) if not tracesuccess or buffer_full: print_error(tag, ("trace buffer filled up before " "tracing turned off - considering this an error " "here").format()) success = False target_pids = [] if not USE_MANUAL_CLIENT: stop_client(client, xvfb_p) service_cleanup(stdouts + stderrs) if manualservice: print_warning(tag, ("target_pids is currently {}, but " "this is not correct! Fill in the target_pids file " "with the correct pid manually.").format(target_pids)) return target_pids
def memcached_exec(outputdir): tag = "memcached_exec" target_pids = [] tracer = traceinfo(MC_APPNAME) (mc_stdout, mc_stderr) = new_out_err_files(outputdir, MC_APPNAME) (client_stdout, client_stderr) = new_out_err_files(outputdir, WHICHCLIENT) success = tracer.trace_on(outputdir, descr=("starting memcached " "server").format()) if not success: print_error(tag, ("trace_on failed, returning now")) close_files([mc_stdout, mc_stderr, client_stdout, client_stderr]) return [] (success, mc_p) = start_mc_server(mc_stdout, mc_stderr) if not success or not mc_p: print_error(tag, ("start_mc_server failed ({}, {}), returning " "now").format(success, mc_p)) close_files([mc_stdout, mc_stderr, client_stdout, client_stderr]) return [] mc_server_pid = mc_p.pid target_pids.append(mc_server_pid) # Start the memcached client: note that for some clients, the # client runs entirely in this method, so success may be returned # but client_p may be None. (success, client_p) = start_mc_client(client_stdout, client_stderr, tracer, mc_p.pid, "load", outputdir) if not success: print_error( tag, ( "start_mc_client failed ({}, {}), stopping " "memcached server and returning. echo {} > target_pids " "if you still want to analyze this trace" ).format(success, client_p, mc_p.pid), ) client_p = None target_pids = [] use_getput_phase = True if success and use_getput_phase: tracer.trace_checkpoint("getput start") (success, client_p) = start_mc_client(client_stdout, client_stderr, tracer, mc_p.pid, "getput", outputdir) if not success: print_error( tag, ("start_mc_client failed ({}, {}), stopping " "memcached server and returning").format( success, client_p ), ) stop_mc_server(mc_p) close_files([mc_stdout, mc_stderr, client_stdout, client_stderr]) return [] elif success: success = run_mc_client(client_p, tracer, mc) if not success: print_error(tag, "run_mc_client failed") target_pids = [] if client_p: stop_mc_client(client_p) stop_mc_server(mc_p) (tracesuccess, buffer_full) = tracer.trace_off("stopping memcached server") if not tracesuccess or buffer_full: print_error( tag, ( "trace buffer filled up before " "tracing turned off - considering this an error " "here. echo {} > target_pids to analyze anyway" ).format(mc_server_pid), ) success = False target_pids = [] close_files([mc_stdout, mc_stderr, client_stdout, client_stderr]) print_debug(tag, ("returning target_pids={}").format(target_pids)) return target_pids
def runservice_manualclient(outputdir, service): tag = 'runservice_manualclient' if service not in KNOWN_SERVICES: print_error(tag, ("service {} not in KNOWN_SERVICES {}, " "returning empty target_pids now").format(service, KNOWN_SERVICES)) return [] target_pids = [] tracer = traceinfo(service) (service_stdout, service_stderr) = run.stdout_stderr_init( outputdir, service) (success, meh) = service_cmd(service, 'stop', outputdir, service_stdout, service_stderr) if not success: print_error(tag, ("initial {} stop failed, returning [] " "now").format(service)) for f in [service_stdout, service_stderr]: f.close() return [] success = tracer.trace_on(outputdir, descr="starting {}".format(service)) if not success: print_error(tag, ("trace_on failed, returning [] now")) for f in [service_stdout, service_stderr]: f.close() return [] (success, service_pid) = service_cmd(service, 'start', outputdir, service_stdout, service_stderr) if service_pid < 2: success = False # If service start didn't succeed, we'll skip the client execution, # but we'll still try to stop the service and then turn tracing # off. if success: # Pause until Ctrl-C (SIGINT) is received. Call a nop signal # handler when signal is received, then reset signal behavior # back to default. # http://docs.python.org/3/library/signal.html signal.signal(signal.SIGINT, run.signal_handler_nop) print(("Tracing is on and {} service is started").format(service)) print(("Run your client, then press Ctrl-C to stop the " "service and disable tracing")) signal.pause() # Note: Linux-only signal.signal(signal.SIGINT, signal.SIG_DFL) success = True service_cmd(service, 'stop', outputdir, service_stdout, service_stderr) # Stop trace *after* stopping service. (tracesuccess, buffer_full) = tracer.trace_off( "stopping {}".format(service), service_pid) if not tracesuccess: print_error(tag, ("trace_off failed")) success = False elif buffer_full: print_error(tag, ("trace buffer filled up before " "tracing turned off - considering this an error " "here").format()) success = False for f in [service_stdout, service_stderr]: f.close() if success: print_debug(tag, ("everything ran successfully, appending " "service_pid {} to target_pids and returning").format( service_pid)) target_pids.append(service_pid) else: print_error(tag, ("something failed, so not appending " "service_pid {} to target_pids").format(service_pid)) target_pids = [] return target_pids