def setup_run_directory(self, full_data_dir, this_run_dir, data_dir, appargs_subdir): if not os.path.isdir(this_run_dir): os.makedirs(this_run_dir) files_to_copy_to_run_dir = glob.glob(os.path.join(full_data_dir, "*.ptx")) +\ glob.glob(os.path.join(full_data_dir, "*.cl")) +\ glob.glob(os.path.join(full_data_dir, "*.h")) +\ glob.glob(os.path.dirname(self.config_file) + "/*.icnt") +\ glob.glob(os.path.dirname(self.config_file) + "/*.xml") for file_to_cp in files_to_copy_to_run_dir: new_file = os.path.join( this_run_dir, os.path.basename(this_directory + file_to_cp)) if os.path.isfile(new_file): os.remove(new_file) shutil.copyfile(file_to_cp, new_file) # link the data directory benchmark_data_dir = os.path.join(full_data_dir, "data") if os.path.isdir(benchmark_data_dir): if os.path.lexists(os.path.join(this_run_dir, "data")): os.remove(os.path.join(this_run_dir, "data")) os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data")) # link the traces directory if options.trace_dir != "": ### This code handles the case where you pass a directory a few levels up from the ### directory where the traces are laid out. benchmark_trace_dir = None paths_to_try = [os.path.join(options.trace_dir, appargs_subdir, "traces")] + \ glob.glob(os.path.join(options.trace_dir, "**", "**", appargs_subdir, "traces")) +\ glob.glob(os.path.join(options.trace_dir, "**", appargs_subdir, "traces")) for path in paths_to_try: try: benchmark_trace_dir = \ common.dir_option_test(path,"", this_directory) break except common.PathMissing as e: pass if benchmark_trace_dir == None: sys.exit("Cannot find traces in any of the paths: {0}".format( paths_to_try)) benchmark_trace_dir = os.path.abspath(benchmark_trace_dir) if os.path.isdir(benchmark_trace_dir): if os.path.lexists(os.path.join(this_run_dir, "traces")): os.remove(os.path.join(this_run_dir, "traces")) os.symlink(benchmark_trace_dir, os.path.join(this_run_dir, "traces")) all_data_link = os.path.join(this_run_dir, "data_dirs") if os.path.lexists(all_data_link): os.remove(all_data_link) if os.path.exists(os.path.join(this_directory, data_dir)): os.symlink(os.path.join(this_directory, data_dir), all_data_link)
exit( "Could not find logfiles for job with the name \"{0}\"".format( options.sim_name)) all_logfiles = named_sim parsed_logfiles.append(max(all_logfiles, key=os.path.getmtime)) elif options.logfile == "all": parsed_logfiles = [os.path.join(logfiles_directory, f) \ for f in os.listdir(logfiles_directory) if(re.match(r'sim_log.*\.latest',f))] else: parsed_logfiles.append( common.file_option_test(options.logfile, "", this_directory)) print "Using logfiles " + str(parsed_logfiles) options.run_dir = common.dir_option_test( options.run_dir, this_directory + ("../../sim_run_%s/" % cuda_version), this_directory) if not os.path.isdir(options.run_dir): exit( options.run_dir + " does not exist - specify the run directory where the benchmark/config dirs exist" ) # Searches the output file for these strings. # If they exist, it assumes that the functional # Test implemented in the CPU program has passed # Searches the output file and the stderr file for these strings # If they exists, then something has gone wrong. status_strings = { "passed": "FUNC_TEST_PASSED",
help="File that contains regex expressions on each line for what apps should be excluded." +\ " Useful for removing random toy apps from the correlation.") parser.add_option("-n", "--noanno", dest="noanno", action="store_true", help="Turn off plot annotations") parser.add_option("-p", "--plotname", dest="plotname", default="", help="string put in the middle of the output files. If nothing is provided, then" +\ "a concatination of all the configs in the graph are used.") (options, args) = parser.parse_args() common.load_defined_yamls() benchmarks = [] options.hardware_dir = common.dir_option_test(options.hardware_dir, "../../run_hw/", this_directory) options.data_mappings = common.file_option_test(options.data_mappings, "correl_mappings.py", this_directory) options.blacklist = common.file_option_test(options.blacklist, "", this_directory) blacklist = [] if options.blacklist != "": for bannedname in open(options.blacklist).readlines(): bannedname = bannedname.strip() blacklist.append(re.compile(bannedname)) logger = Logger(options.verbose) # Get the hardware Data
if str(os.getenv("GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN")) != "1": sys.exit("ERROR - Please run setup_environment before running this script") cuda_version = common.get_cuda_version(this_directory) if options.run_directory == "": options.run_directory = os.path.join(this_directory, "../../sim_run_%s" % cuda_version) else: options.run_directory = os.path.join(os.getcwd(), options.run_directory) # Let's copy out the .so file so that builds don't interfere with running tests # If the user does not specify a so file, then use the one in the git repo and copy it out. options.so_dir = common.dir_option_test( options.so_dir, os.path.join(os.getenv("GPGPUSIM_ROOT"), "lib", os.getenv("GPGPUSIM_CONFIG")), this_directory) if options.trace_dir == "": so_path = os.path.join(options.so_dir, "libcudart.so") else: so_path = os.path.join(options.so_dir, "gpgpusim.out") version_string = extract_version(so_path) running_so_dir = os.path.join(options.run_directory, "gpgpu-sim-builds", version_string) if not os.path.exists(running_so_dir): # In the very rare case that concurrent builds try to make the directory at the same time # (after the test to os.path.exists -- this has actually happened...) try: os.makedirs(running_so_dir) except:
def run(self, build_handle, benchmarks, run_directory, cuda_version, simdir): for dir_bench in benchmarks: exec_dir, data_dir, benchmark, self.command_line_args_list = dir_bench if options.trace_dir == "": full_exec_dir = common.dir_option_test( os.path.expandvars(exec_dir), "", this_directory) full_data_dir = common.dir_option_test(\ os.path.join(os.path.expandvars(data_dir), benchmark.replace('/','_')), "",\ this_directory) else: full_exec_dir = "" # For traces it is not necessary to have the apps built full_data_dir = "" self.benchmark_args_subdirs = {} for args in self.command_line_args_list: self.benchmark_args_subdirs[args] = common.get_argfoldername( args) for args in self.command_line_args_list: appargs_run_subdir = os.path.join( benchmark.replace('/', '_'), self.benchmark_args_subdirs[args]) this_run_dir = os.path.join(run_directory, appargs_run_subdir, self.run_subdir) self.setup_run_directory(full_data_dir, this_run_dir, data_dir, appargs_run_subdir) self.text_replace_torque_sim(full_data_dir, this_run_dir, benchmark, cuda_version, args, simdir, full_exec_dir, build_handle) self.append_gpgpusim_config(benchmark, this_run_dir, self.config_file) # Submit the job to torque and dump the output to a file if not options.no_launch: torque_out_filename = this_directory + "torque_out.{0}.txt".format( os.getpid()) torque_out_file = open(torque_out_filename, 'w+') saved_dir = os.getcwd() os.chdir(this_run_dir) if subprocess.call([job_submit_call,\ os.path.join(this_run_dir , job_template)],\ stdout=torque_out_file) < 0: exit("Error Launching Job") else: # Parse the torque output for just the numeric ID torque_out_file.seek(0) torque_out = re.sub(r"[^\d]*(\d*).*", r"\1", torque_out_file.read().strip()) print("Job " + torque_out + " queued (" +\ benchmark + "-" + self.benchmark_args_subdirs[args] +\ " " + self.run_subdir + ")") torque_out_file.close() os.remove(torque_out_filename) os.chdir(saved_dir) if len(torque_out) > 0: # Dump the benchmark description to the logfile if not os.path.exists(this_directory + "logfiles/"): # In the very rare case that concurrent builds try to make the directory at the same time # (after the test to os.path.exists -- this has actually happened...) try: os.makedirs(this_directory + "logfiles/") except: pass now_time = datetime.datetime.now() day_string = now_time.strftime("%y.%m.%d-%A") time_string = now_time.strftime("%H:%M:%S") log_name = "sim_log.{0}".format(options.launch_name) logfile = open(this_directory +\ "logfiles/"+ log_name + "." +\ day_string + ".txt",'a') print >> logfile, "%s %6s %-22s %-100s %-25s %s.%s" %\ ( time_string ,\ torque_out ,\ benchmark ,\ self.benchmark_args_subdirs[args] ,\ self.run_subdir,\ benchmark,\ build_handle ) logfile.close() self.benchmark_args_subdirs.clear()
if str(os.getenv("GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN")) != "1": sys.exit("ERROR - Please run setup_environment before running this script") cuda_version = common.get_cuda_version(this_directory) if options.run_directory == "": options.run_directory = os.path.join(this_directory, "../../sim_run_%s" % cuda_version) else: options.run_directory = os.path.join(os.getcwd(), options.run_directory) # Let's copy out the .so file so that builds don't interfere with running tests # If the user does not specify a so file, then use the one in the git repo and copy it out. if options.trace_dir == "": options.simulator_dir = common.dir_option_test( options.simulator_dir, os.path.join(os.getenv("GPGPUSIM_ROOT"), "lib", os.getenv("GPGPUSIM_CONFIG")), this_directory) simulator_path = os.path.join(options.simulator_dir, "libcudart.so") else: options.simulator_dir = common.dir_option_test( options.simulator_dir, os.path.join(os.getenv("ACCELSIM_ROOT"), "bin", os.getenv("ACCELSIM_CONFIG")), this_directory) simulator_path = os.path.join(options.simulator_dir, "accel-sim.out") version_string = extract_version(simulator_path) running_sim_dir = os.path.join(options.run_directory, "gpgpu-sim-builds", version_string) if not os.path.exists(running_sim_dir): # In the very rare case that concurrent builds try to make the directory at the same time # (after the test to os.path.exists -- this has actually happened...)
for bench in benchmarks: edir, ddir, exe, argslist = bench for argpair in argslist: args = argpair["args"] run_name = os.path.join(exe, common.get_argfoldername(args)) this_run_dir = os.path.join(this_directory, "..", "..", "hw_run", "device-" + options.device_num, cuda_version, run_name) if not os.path.exists(this_run_dir): os.makedirs(this_run_dir) # link the data directory try: benchmark_data_dir = common.dir_option_test( os.path.join(ddir, exe, "data"), "", this_directory) if os.path.lexists(os.path.join(this_run_dir, "data")): os.remove(os.path.join(this_run_dir, "data")) os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data")) except common.PathMissing: pass all_data_link = os.path.join(this_run_dir, "data_dirs") if os.path.lexists(all_data_link): os.remove(all_data_link) top_data_dir_path = common.dir_option_test(ddir, "", this_directory) os.symlink(top_data_dir_path, all_data_link) if args == None: args = ""
common.load_defined_yamls() benchmarks = [] benchmarks = common.gen_apps_from_suite_list(options.benchmark_list.split(",")) cuda_version = common.get_cuda_version(this_directory) now_time = datetime.datetime.now() day_string = now_time.strftime("%y.%m.%d-%A") time_string = now_time.strftime("%H:%M:%S") logfile = day_string + "--" + time_string + ".csv" nvbit_tracer_path = os.path.join(this_directory, "tracer_tool") for bench in benchmarks: edir, ddir, exe, argslist = bench ddir = common.dir_option_test(os.path.join(ddir, exe), "", this_directory) edir = common.dir_option_test(edir, "", this_directory) for args in argslist: run_name = os.path.join(exe, common.get_argfoldername(args)) this_run_dir = os.path.abspath( os.path.expandvars( os.path.join(this_directory, "..", "..", "hw_run", "traces", "device-" + options.device_num, cuda_version, run_name))) this_trace_folder = os.path.join(this_run_dir, "traces") if not os.path.exists(this_run_dir): os.makedirs(this_run_dir) if not os.path.exists(this_trace_folder): os.makedirs(this_trace_folder)
def main(): parser = OptionParser() parser.add_option("-a", "--apps", dest="apps", default = None, help="Pass the comma seperated input list instead of asking "\ "from stdin. Example: -a tesla-v100/rodinia-3.1,tesla-v100/cudasdk") parser.add_option("-d", "--download_dir", dest="download_dir", default="", help="Directory to download the traces to.") (options, args) = parser.parse_args() try: hw_run_dir = common.dir_option_test(options.download_dir, "hw_run", this_directory) except common.PathMissing: hw_run_dir = os.path.join(this_directory, "hw_run") if not os.path.exists(hw_run_dir): os.makedirs(hw_run_dir) os.chdir(hw_run_dir) # Parse the trace summary trace_summary = os.path.join(hw_run_dir, VERSION + ".trace.summary.txt") try: os.remove(trace_summary) except OSError: pass Popen(["wget " + " " + WEB_DIRECTORY + VERSION + ".trace.summary.txt"], stderr=STDOUT, shell=True).communicate() lineFormat = re.compile(r"(.*)\t(.*)/" + VERSION + ".latest/(.*)") sizeDict = {} for line in open(trace_summary): lineMatch = lineFormat.match(line) if lineMatch != None: size = lineMatch.group(1) cardName = lineMatch.group(2) fileName = lineMatch.group(3) suiteName = re.sub(r"(.*)\.tgz", r"\1", fileName) if cardName not in sizeDict: sizeDict[cardName] = Card(cardName) card = sizeDict[cardName] if suiteName not in card.suites: newSuite = Suite(suiteName) card.suites[suiteName] = Suite(suiteName) suite = sizeDict[cardName].suites[suiteName] if suiteName != fileName: suite.compressedSize = getNumRaw(size) else: suite.uncompressedSize = getNumRaw(size) # Infor the user what is available - ask them what they want to do print "\n\nCurrently Available Traces:" for cardName, card in sizeDict.iteritems(): print "GPU Name: {0}. All Apps Compressed (Download size): {1}, All Apps Uncompressed (Size on disk when used): {2}"\ .format(card.name, millify(card.getTotalCompressed()), millify(card.getTotalUncompressed())) for name, suite in card.suites.iteritems(): print "\t{0}: Compressed = {1}, Uncompressed = {2}".format(suite.name,\ millify(suite.compressedSize), millify(suite.uncompressedSize)) selectionValid = False while not selectionValid: if options.apps == None: selection = raw_input("\n-------\nWhat do you want to download?"\ "\n<card/suite>,<card/suite> (i.e. tesla-v100/rodinia-3.1,tesla-v100/cudasdk)"\ "\n(Default=all/all) : ") if selection == "" or selection == None: selection = "all/all" else: selection = options.apps try: for item in selection.split(","): cardName = item.split(r"/")[0] suiteName = item.split(r"/")[1] if cardName == "all": for cardName, card in sizeDict.iteritems(): if suiteName == "all": for suiteName, suite in card.suites.iteritems(): downloadTrace(cardName, suiteName) else: downloadTrace(cardName, suiteName) else: card = sizeDict[cardName] if suiteName == "all": for suiteName, suite in card.suites.iteritems(): downloadTrace(cardName, suiteName) else: downloadTrace(cardName, suiteName) selectionValid = True except Exception as e: selectionValid = False print "Invalid Input: {0}".format(e) if options.apps != None: sys.exit(1) print "\n\nDownload successful to {0}.\nFiles must be uncompressed with tar -xzvf <filename> to be usable by accel-sim"\ .format(hw_run_dir)