def start_opal(transactions, dump_interval, opal_outfile):
env_dict = workloads.prepare_env_dictionary(simics = 1)
mfacet.run_sim_command("load-module opal")
# set the SMT thread count
smt_threads = int(workloads.get_var(env_dict, "SMT_THREADS"))
mfacet.run_sim_command("opal0.setparam CONFIG_LOGICAL_PER_PHY_PROC %d" % smt_threads)
# Register file should scale with SMT thread count
mfacet.run_sim_command("opal0.setparam CONFIG_IREG_PHYSICAL %d" % (160*smt_threads+128))
mfacet.run_sim_command("opal0.setparam CONFIG_FPREG_PHYSICAL %d" % (64*smt_threads+256))
mfacet.run_sim_command("opal0.setparam CONFIG_CCREG_PHYSICAL %d" % (5*smt_threads+128))
mfacet.run_sim_command("opal0.setparam CONFIG_NUM_CONTROL_SETS %d" % (64*smt_threads))
# do Transactional loads/stores at retirment, not speculatively
mfacet.run_sim_command("opal0.setparam RETIREMENT_CACHE_ACCESS %d" % 1)
# ROB, scheduling window sizes
mfacet.run_sim_command("opal0.setparam IWINDOW_ROB_SIZE %d" % 128)
mfacet.run_sim_command("opal0.setparam IWINDOW_WIN_SIZE %d" % 64)
mfacet.run_sim_command("opal0.setparam RESERVED_ROB_ENTRIES %d" % 4)
# Use HW or SW log
mfacet.run_sim_command("opal0.setparam_str XACT_USE_SWLOG true")
mfacet.run_sim_command("opal0.setparam XACT_MAX_DEPTH %d" % string.atoi(workloads.get_var(env_dict, "MAX_DEPTH")));
# Comment this out for production/faster runs
#mfacet.run_sim_command("opal0.setparam_str XACT_ISOLATION_CHECK true")
mfacet.run_sim_command('opal0.init')
print "starting opal"
mfacet.run_sim_command('opal0.sim-start %s' % opal_outfile)
# now setup and run
mfacet.setup_run_for_n_transactions(transactions, dump_interval)
mfacet.run_sim_command("opal0.sim-step 100000000000")
for (protocol, protocol_option) in config.protocol_list:
for opal_config in config.opal_config_file_list:
print " Creating %s, %s" % (module, protocol_option)
for bandwidth in config.bandwidth_list:
condor_file.write("## run: %s module, %s opal, %d chips, %d procs per chip, %d smt threads, %s protocol option, %d bandwidth\n" %
(protocol, opal_config, chips, procs_per_chip, smt_threads, protocol_option, bandwidth))
condor_file.write("\n")
req = string.join(requirement_list, " && ")
condor_file.write("requirements = %s\n" % req)
condor_file.write("\n")
## Environment - this is how we pass parameters to simics
env_dict = workloads.prepare_env_dictionary(simics = 0)
if config.random_seed:
workloads.set_var(env_dict, 'RANDOM_SEED', config.random_seed)
workloads.set_var(env_dict, 'OPAL_CONFIG_NAME', opal_config[0])
workloads.set_var(env_dict, 'OPAL_CONFIG_FILE', opal_config[1])
# for the checkpoint, we have to extract the checkpoint storage directory
# from the checkpoints configuration file
lines = grep("../checkpoints/workload.conf", "WORKLOAD_CHECKPOINT_DIR=")
if(len(lines) != 1):
print "The workload checkpoint directory could not be determined"
os.exit(1)
workload_check_dir = lines[0].split("=")[1]
if protocol_option:
def run_simics(
checkpoint,
workload_name,
transactions,
protocol={"name": "test"},
processors=16,
smt_threads=1,
phy_signature="Perfect_",
virtual_signature="Perfect_",
summary_signature="Perfect_",
xact_max_depth="1",
microbenchmark=0,
mbench_arg_prefix="",
mbench_arg_string="",
bench_arg=0,
condor_process=os.getpid(),
condor_cluster=os.getppid(),
expected_ruby_cycles=0,
tolerance=.05,
check_opal=0,
check_ruby=0,
):
# use SPARC V8 directory
name = protocol["name"]
output = tools.run_command(
"scripts/prepare_simics_home.sh simics/home/%s" % name)
if protocol.has_key("specific_processor_count"):
processors = protocol.get("specific_processor_count", 1)
procs_per_chip = protocol.get("procs_per_chip", 1)
l2_cache_banks = protocol.get("specific_cache_count",
(processors / smt_threads))
eager_cd = protocol.get("eager_cd", 1)
eager_vm = protocol.get("eager_vm", 1)
magic_waiting = protocol.get("magic_waiting", 0)
no_backoff = protocol.get("no_backoff", 0)
bandwidth = protocol.get("bandwidth", 6400)
if procs_per_chip > processors:
procs_per_chip = 1
log_message(
"Running simics: checkpoint=%s, processors=%s, smt_threads=%s, procs_per_chip=%d transactions=%d, protocol: %s"
% (checkpoint, processors, smt_threads, procs_per_chip, transactions,
name))
# prepare environment variables for running simics
env_dict = workloads.prepare_env_dictionary(simics=0)
workloads.set_var(env_dict, "RESULTS_DIR", "../../../results")
workloads.set_var(env_dict, "WORKLOAD", workload_name)
workloads.set_var(env_dict, "CHECKPOINT", checkpoint)
workloads.set_var(env_dict, "CHECKPOINT_DIR", " ")
workloads.set_var(env_dict, "PROTOCOL", name)
workloads.set_var(env_dict, "PROCESSORS", processors)
workloads.set_var(env_dict, "CHIPS", 1)
workloads.set_var(env_dict, "SMT_THREADS", smt_threads)
workloads.set_var(env_dict, "PROCS_PER_CHIP", procs_per_chip)
workloads.set_var(env_dict, "NUM_L2_BANKS", l2_cache_banks)
workloads.set_var(env_dict, "TRANSACTIONS", transactions)
workloads.set_var(env_dict, "BANDWIDTH", bandwidth)
if (g_random_seed != "random"):
workloads.set_var(env_dict, "RANDOM_SEED", g_random_seed)
workloads.set_var(env_dict, "CONDORCLUSTER", condor_cluster)
workloads.set_var(env_dict, "CONDORPROCESS", condor_process)
# Transactional Memory variables
workloads.set_var(env_dict, "MICROBENCH_DIR",
"microbenchmarks/transactional")
workloads.set_var(env_dict, "BENCHMARK", workload_name)
workloads.set_var(env_dict, "MAX_DEPTH", xact_max_depth)
workloads.set_var(env_dict, 'LOCK_TYPE', "TM")
workloads.set_var(env_dict, 'READ_WRITE_FILTER', phy_signature)
workloads.set_var(env_dict, 'VIRTUAL_READ_WRITE_FILTER', virtual_signature)
workloads.set_var(env_dict, 'SUMMARY_READ_WRITE_FILTER', summary_signature)
workloads.set_var(env_dict, "XACT_EAGER_CD", eager_cd)
if (eager_vm == 0):
workloads.set_var(env_dict, "XACT_LAZY_VM", 1)
else:
workloads.set_var(env_dict, "XACT_LAZY_VM", 0)
workloads.set_var(env_dict, "ENABLE_MAGIC_WAITING", magic_waiting)
workloads.set_var(env_dict, "XACT_NO_BACKOFF", no_backoff)
# set per-microbenchmark specific variables
#if ((workload_name == "compensation") or (workload_name == "commit-action")
# or (workload_name == "isolation-test") or (workload_name == "logging-test")
# or (workload_name == "partial-rollback")):
# workloads.set_var(env_dict, "MICROBENCH_DIR", "microbenchmarks/transactional/test")
workloads.set_var(env_dict, 'MBENCH_ARG_PREFIX', mbench_arg_prefix)
workloads.set_var(env_dict, 'MBENCH_ARG_STRING', mbench_arg_string)
workloads.update_system_env(env_dict)
# create results directory
output = tools.run_command("/bin/rm -rf results")
mbench_arg_prefix = workloads.get_var(env_dict, 'MBENCH_ARG_PREFIX')
os.mkdir("results")
if (microbenchmark == 1):
print "CREATING DIRECTORY results/%s-%s" % (workload_name,
mbench_arg_prefix)
os.mkdir("results/%s-%s" % (workload_name, mbench_arg_prefix))
else:
os.mkdir("results/" + workload_name)
print "WORKLOAD NAME %s" % workload_name
#'''
os.chdir("simics/home/%s/" % name)
# run the microbenchmark script if needed
if (microbenchmark == 1):
output = tools.run_command(
"./simics -echo -verbose -no-log -no-win -x ../../../gen-scripts/microbench.simics",
"quit 666\n",
verbose=1,
max_lines=0)
else:
output = tools.run_command(
"./simics -echo -verbose -no-log -no-win -x ../../../gen-scripts/go.simics",
"quit 666\n",
verbose=1)
#tools.run_command("./simics -echo -verbose -no-log -x ../../../gen-scripts/go.simics", "quit 666\n")
os.chdir("../../..")
# dump simics output
if (microbenchmark == 0):
simics_output_filename = "results/%s.output" % workloads.get_output_file_name_prefix(
env_dict)
else:
simics_output_filename = "results/%s.output" % workloads.get_microbench_output_file_name_prefix(
env_dict, 0)
simics_output = open(simics_output_filename, "w")
simics_output.write(output)
simics_output.close()
if check_ruby == 1 and name != "template":
if (microbenchmark == 0):
ruby_stats_filename = "results/%s.stats" % workloads.get_output_file_name_prefix(
env_dict)
error_output_filename = "condor/results/%s.error" % (
workloads.get_output_file_name_prefix(env_dict, 0))
else:
ruby_stats_filename = "results/%s.stats" % workloads.get_microbench_output_file_name_prefix(
env_dict, 0)
error_output_filename = "condor/results/%s.error" % (
workloads.get_microbench_output_file_name_prefix(env_dict, 0))
if (not os.path.exists(ruby_stats_filename)):
raise RegressionError(
"Ruby stats output file not present: %s" % ruby_stats_filename,
output)
# Check for error file, indicating a SIMICS_ASSERT() failure
if (os.path.exists(error_output_filename)):
print "SIMICS ASSERT error!"
raise RegressionError(
"SIMICS_ASSERT error file found: %s" % error_output_filename,
output)
# get random seed
simics_output = open(simics_output_filename, "r")
for line in simics_output.readlines():
if re.search("g_RANDOM_SEED", line):
tokens = line.split()
log_message(" Random seed: %d" % int(tokens[4][:-1]))
# get ruby cycle
ruby_stats = open(ruby_stats_filename, "r")
ruby_cycles = 0
for line in ruby_stats.readlines():
line_elements = string.split(line)
if len(line_elements) > 1 and line_elements[0] == "Ruby_cycles:":
ruby_cycles = int(line_elements[1])
if (ruby_cycles == 0):
raise RegressionError(
"Ruby_cycles not found from the output file: %s" %
ruby_stats_filename, output)
else:
log_message(" Ruby_cycles: %d" % ruby_cycles)
if (expected_ruby_cycles != 0):
percent_diff = 1.0 * ruby_cycles / expected_ruby_cycles
if percent_diff < (1.0 - tolerance) or percent_diff > (1.0 +
tolerance):
log_message(
" Checking ruby_cycles - ratio is %f: OUT OF RANGE" %
percent_diff)
log_error(
"ERROR: Ruby_cycles not within tolerances. expected %d, actual %d"
% (expected_ruby_cycles, ruby_cycles))
else:
log_message(" Checking ruby_cycles - ratio is %f: OK" %
percent_diff)
if check_opal == 1:
opal_log_filename = "results/%s.opal" % workloads.get_output_file_name_prefix(
env_dict)
if (not os.path.exists(opal_log_filename)):
raise RegressionError(
("Opal log file not present: %s" % opal_log_filename), output)
# check opal correct rate!
else:
opal_log = open(opal_log_filename)
processor_total_instructions = 1001 # > 1000
processor_correct_rate = 98 # < 99
for line in opal_log.readlines():
tokens = line.split()
# remember the correct rate
if (len(tokens) == 5 and tokens[1] == "Percent"
and tokens[2] == "correct"):
processor_correct_rate = float(tokens[4])
# remember the processor's commit instruction number
if (len(tokens) == 6 and tokens[1] == "Total"
and tokens[2] == "number" and tokens[3] == "of"
and tokens[4] == "instructions"):
processor_total_instructions = int(tokens[5])
# check the correct rate here since the total instruction
# number comes last during the scan of the output file
if (processor_correct_rate < 99
and processor_total_instructions > 1000):
raise RegressionError((
"Opal correct rate too low (%f%% of %d instructions)!"
% (processor_correct_rate,
processor_total_instructions)), output)
def load_modules_and_run(fast_forward = 0):
print "Python version: %s" % sys.version
###### Read simulation parameters
env_dict = workloads.prepare_env_dictionary(simics = 1)
workloads.check_requirements(env_dict)
workloads.print_all_variables(env_dict)
protocol = workloads.get_var(env_dict, "PROTOCOL")
workload_name = workloads.get_var(env_dict, "WORKLOAD")
checkpoint_dir = workloads.get_var(env_dict, "CHECKPOINT_DIR")
checkpoint = workloads.get_var(env_dict, "CHECKPOINT")
chips = int(workloads.get_var(env_dict, "CHIPS"))
processors_per_chip = int(workloads.get_var(env_dict, "PROCS_PER_CHIP"))
smt_threads = int(workloads.get_var(env_dict, "SMT_THREADS"))
l2_banks = int(workloads.get_var(env_dict, "NUM_L2_BANKS"))
bandwidth = int(workloads.get_var(env_dict, "BANDWIDTH"))
results_dir = workloads.get_var(env_dict, "RESULTS_DIR")
transactions = int(workloads.get_var(env_dict, "TRANSACTIONS"))
dump_interval = int(workloads.get_var(env_dict, "DUMP_INTERVAL"))
condor_cluster = int(workloads.get_var(env_dict, "CONDORCLUSTER"))
condor_process = int(workloads.get_var(env_dict, "CONDORPROCESS"))
checkpoint_at_end = workloads.get_var(env_dict, "CHECKPOINT_AT_END")
generate_request_trace = workloads.get_var(env_dict, "GENERATE_TRACE")
generate_cache_data_dump = workloads.get_var(env_dict, "CACHE_DATA_DUMP")
protocol_option = workloads.get_var(env_dict, "PROTOCOL_OPTION")
network_topology = workloads.get_var(env_dict, "NETWORK_TOPOLOGY")
warmup_file = workloads.get_var(env_dict, "WARMUP_FILE")
opal_config_file = workloads.get_var(env_dict, "OPAL_CONFIG_FILE")
opal_config_name = workloads.get_var(env_dict, "OPAL_CONFIG_NAME")
interactive = workloads.get_var(env_dict, "INTERACTIVE")
random_seed = workloads.get_var(env_dict, "RANDOM_SEED")
use_local_mirror = workloads.get_var(env_dict, "USE_LOCAL_MIRROR")
if random_seed == None:
random_seed = (condor_cluster << 16) | condor_process
else:
random_seed = int(random_seed)
###### mandatory parameters
assert(protocol != None)
assert(workload_name != None)
assert(checkpoint != None)
assert(chips != None)
assert(processors_per_chip != None)
assert(smt_threads != None)
assert(results_dir != None)
assert(transactions != None)
assert(dump_interval != None)
assert(condor_cluster != None)
assert(condor_process != None)
ruby = None
if "ruby" in get_module_list():
ruby = 1 # true
opal = None
if "opal" in get_module_list():
opal = 1 # true
###### print out local host name to help with troubleshooting
print "Local host name:", string.strip(os.popen("hostname").readline())
###### init simics with a checkpoint
assert(checkpoint[0] == '/')
run_sim_command('read-configuration "%s/%s"' % (checkpoint_dir, checkpoint))
if(fast_forward): run_sim_command("continue %d" % fast_forward)
###### Conserve memory by limiting how much memory each simics image object can use
#@mfacet.limit_all_images(256)
###### set simics parameters
# enable/disable STCs here
run_sim_command("istc-disable")
run_sim_command("dstc-disable")
run_sim_command("stc-status")
# disable breakpoint before we sync all processors
run_sim_command('magic-break-disable')
# always use 1 for better parallelism
old_switch_time = conf.sim.cpu_switch_time
print "Old cpu_switch_time: %d" % conf.sim.cpu_switch_time
conf.sim.cpu_switch_time = 1
# 8/28/2003 CM: stepping or continuing simics before loading opal results
# in a major performance loss. See me for more details.
if not opal:
run_sim_command("c %d"%(10*old_switch_time)) # sync CPUs after changing switch time
print "New cpu_switch_time: %d" % conf.sim.cpu_switch_time
# enable breakpoint
run_sim_command('magic-break-enable')
# enable IFETCH, i-fetch line size is controlled in checkpoint files
run_sim_command("instruction-fetch-mode instruction-fetch-trace")
run_sim_command("instruction-fetch-mode")
filename_prefix = "%s/%s" % (results_dir, workloads.get_output_file_name_prefix(env_dict, 1))
prep_dir(filename_prefix)
###### capture xterm output in file
import mod_xterm_console_commands
mod_xterm_console_commands.cap_start_cmd(get_console(), filename_prefix + ".xterm")
###### Load modules and set parameters
if opal:
run_sim_command('load-module opal')
if ruby:
run_sim_command("load-module ruby")
if network_topology:
run_sim_command('ruby0.setparam_str g_NETWORK_TOPOLOGY "%s"' % network_topology)
if protocol_option:
run_sim_command('ruby0.setparam_str g_MASK_PREDICTOR_CONFIG "%s"' % protocol_option)
run_sim_command("ruby0.setparam g_PROCS_PER_CHIP %d" % processors_per_chip)
run_sim_command("ruby0.setparam g_NUM_L2_BANKS %d" % l2_banks)
run_sim_command("ruby0.setparam g_RANDOM_SEED %d" % random_seed)
run_sim_command("ruby0.setparam g_endpoint_bandwidth %d" % bandwidth)
run_sim_command("ruby0.setparam g_NUM_PROCESSORS %d" % (chips*processors_per_chip))
run_sim_command("ruby0.setparam g_NUM_SMT_THREADS %d" % smt_threads)
transactional_memory = 0
if transactional_memory:
run_sim_command("ruby0.setparam_str REMOVE_SINGLE_CYCLE_DCACHE_FAST_PATH true")
run_sim_command("ruby0.setparam_str PROFILE_EXCEPTIONS true")
run_sim_command("ruby0.setparam_str PROFILE_XACT true")
run_sim_command("ruby0.setparam_str XACT_MEMORY true")
run_sim_command("ruby0.setparam XACT_MAX_DEPTH %s" % xact_max_depth);
#run_sim_command("ruby0.setparam_str DATA_BLOCK true")
run_sim_command("ruby0.setparam RETRY_LATENCY 10")
#run_sim_command("ruby0.setparam RANDOM_SEED 0")
run_sim_command("ruby0.setparam g_MEMORY_SIZE_BYTES 8589934592")
run_sim_command("ruby0.setparam g_DEADLOCK_THRESHOLD 20000000")
run_sim_command("ruby0.init")
run_sim_command("ruby0.periodic-stats-interval %d" % 1000000)
run_sim_command('ruby0.periodic-stats-file "%s.periodic" ' % filename_prefix)
debug = 0
if debug:
run_sim_command('ruby0.debug-verb high')
run_sim_command('ruby0.debug-filter lseagTSN')
#run_sim_command('ruby0.debug-start-time "500000"')
run_sim_command('ruby0.debug-start-time "0"')
###### init opal
if opal:
# For debugging with symtable, used by Min's race detector
#run_sim_command('load-module symtable')
#run_sim_command('new-symtable mysql')
# set up the number of SMT Threads per physical processor::
run_sim_command("opal0.setparam CONFIG_LOGICAL_PER_PHY_PROC %d" % smt_threads)
# set up the number of physical registers needed, for each physical register type:
run_sim_command("opal0.setparam CONFIG_IREG_PHYSICAL %d" % (160*smt_threads+64))
run_sim_command("opal0.setparam CONFIG_FPREG_PHYSICAL %d" % (64*smt_threads+128))
run_sim_command("opal0.setparam CONFIG_CCREG_PHYSICAL %d" % (5*smt_threads+64))
if opal_config_file:
run_sim_command('opal0.init "%s"' % opal_config_file)
else:
run_sim_command('opal0.init')
run_sim_command('opal0.sim-start "%s.opal"' % filename_prefix)
###### do the following only when it is not interactive simulation
if not interactive:
###### warm up the caches
if ruby and warmup_file:
print "Warming caches with %s" % warmup_file
run_sim_command('ruby0.load-caches "%s/%s"' % (checkpoint_dir, warmup_file))
###### generate request trace
if ruby and generate_request_trace:
trace_filename = "/scratch/%s/traces/%s-%dc-%dp-%dt-%s-%d-%d-%d.trace.gz" % (getpass.getuser(), workload_name, chips, processors_per_chip, smt_threads, protocol, bandwidth, condor_cluster, condor_process)
prep_dir(trace_filename)
run_sim_command("ruby0.tracer-output-file %s" % trace_filename)
###### Run for n transactions or just a short run
print "Initial memory usage: %s" % memory_usage_str()
print "Running..."
if transactions > 0:
assert(dump_interval > 0)
setup_run_for_n_transactions(transactions, dump_interval)
if opal:
run_sim_command("opal0.sim-step 100000000000")
else:
run_sim_command("c")
else:
if opal:
run_sim_command("opal0.sim-step 1000")
else:
run_sim_command("c 10000")
###### dump statistics
if ruby:
run_sim_command('ruby0.dump-stats "%s.stats"' % filename_prefix)
if generate_cache_data_dump:
run_sim_command('ruby0.dump-cache-data 0 "%s.cache"' % filename_prefix)
if opal:
run_sim_command('opal0.listparam')
run_sim_command('opal0.stats')
run_sim_command('opal0.sim-stop')
if checkpoint_at_end == 'yes':
mf_write_configuration_cmd("/scratch/%s/%s-%dc-%dp-%dt-%d-caches" % (getpass.getuser(), checkpoint, chips, processors_per_chip, smt_threads, transactions))
# close trace file
if ruby and generate_request_trace == 'yes':
run_sim_command('ruby0.tracer-output-file ""')
###### display resource usage
print_resource_usage()
###### close xterm capture file
get_console().output_file = ""
###### finishing up
run_sim_command('quit')
else:
print "enter interactive mode..."
print "Normally, the following command lines will be issued:"
print ""
print " @from mfacet import *"
print " @setup_run_for_n_transactions(%d, %d) % (transactions, dump_interval)"
print ""
print " # (optionally)"
print " ruby0.load-caches <workload>.warmup.gz"
print " ruby0.tracer-output-file <filename>.trace.gz"
print ""
print " # (if ruby only)"
print " continue"
print " ruby0.dump-stats file.stats"
print ""
print " # (if ruby + opal)"
print " opal0.sim-step 100000000000"
print " opal0.listparam"
print " opal0.stats"
print " opal0.sim-stop"
print ""
print " quit"
def end_transaction_callback(desc, cpu, val):
if (val > 0x10000):
val = val >> 16
global __start_transaction_counter
global __end_transaction_counter
if val == 5: # end of trans
#print "end of transaction of thread %d" % get_tid()
__end_transaction_counter += 1
#print_cpu_regs(cpu)
elif val == 3: # start of trans
#print "start of transaction of thread %d" % get_tid()
__start_transaction_counter += 1
elif val == 4: # break simulation
return
elif val >= 6 and val < 15:
return
elif val == 15:
print "ASSERT FAILURE at PC %x" % SIM_get_program_counter(SIM_current_processor())
# dump a .error file
env_dict = workloads.prepare_env_dictionary(simics = 1)
results_dir = workloads.get_var(env_dict, "RESULTS_DIR")
processors = workloads.get_var(env_dict, "PROCESSORS")
print processors
error_output_filename = "%s/%s.error" % (results_dir, workloads.get_microbench_output_file_name_prefix(env_dict,0))
error_output = open(error_output_filename, "w")
error_output.write("SIMICS ASSERT FAILED at PC %x" % SIM_get_program_counter(SIM_current_processor()))
error_output.close()
SIM_hap_unregister_callback("Core_Magic_Instruction", end_transaction_callback, "end_transaction_magic_call")
try:
opal0 = SIM_get_object( "opal0" )
SIM_set_attribute( opal0, "break_simulation", 1 )
except:
# if opal if not installed, call simics to halt simulation
SIM_break_simulation("%s: %d transactions reached" % (desc, __end_transaction_counter))
elif val >= 16 and val <= 62:
return
elif val >= 1024 and val < 7168:
return
else:
print "%s: Unexpected magic call number %d" % (desc, val)
# dump sim_cycle and sim_inst out
if __transaction_dump_interval != 0 and __end_transaction_counter%__transaction_dump_interval == 0:
# dump the transaction count
print "%s: transaction started: %d, transaction completed: %d, transaction_limit: %d, %s" % (desc, __start_transaction_counter, __end_transaction_counter, __transaction_limit, memory_usage_str())
# if the trace module is installed, dump statistics
try:
trace0 = SIM_get_object("trace0")
SIM_set_attribute(trace0, "dump_stats", __end_transaction_counter);
except:
pass
if __end_transaction_counter >= __transaction_limit:
print "%s: limit reached, unregistering callback" % desc
SIM_hap_unregister_callback("Core_Magic_Instruction", end_transaction_callback, "end_transaction_magic_call")
# if opal is installed, tell it to halt simulation
try:
opal0 = SIM_get_object( "opal0" )
SIM_set_attribute( opal0, "break_simulation", 1 )
except:
# if opal if not installed, call simics to halt simulation
SIM_break_simulation("%s: %d transactions reached" % (desc, __end_transaction_counter))
def start_ruby(debug=0):
env_dict = workloads.prepare_env_dictionary(simics = 1)
condor_cluster = int(workloads.get_var(env_dict, "CONDORCLUSTER"))
condor_process = int(workloads.get_var(env_dict, "CONDORPROCESS"))
random_seed = workloads.get_var(env_dict, "RANDOM_SEED")
if random_seed == None:
import whrandom
#random_seed = whrandom.randint(0,100000)
random_seed = (condor_cluster << 16) | condor_process
else:
random_seed = int(workloads.get_var(env_dict, "RANDOM_SEED"))
smt_threads = int(workloads.get_var(env_dict, "SMT_THREADS"))
processors_per_chip = int(workloads.get_var(env_dict, "PROCS_PER_CHIP"))
read_write_filter = workloads.get_var(env_dict, "READ_WRITE_FILTER")
virtual_signature = workloads.get_var(env_dict, "VIRTUAL_READ_WRITE_FILTER")
summary_signature = workloads.get_var(env_dict, "SUMMARY_READ_WRITE_FILTER")
enable_tourmaline = int(workloads.get_var(env_dict, "ENABLE_TOURMALINE"))
xact_lazy = int(workloads.get_var(env_dict, "XACT_LAZY_VM"))
xact_eager = int(workloads.get_var(env_dict, "XACT_EAGER_CD"))
xact_visualizer = int(workloads.get_var(env_dict, "XACT_VISUALIZER"))
xact_commit_token = int(workloads.get_var(env_dict, "XACT_COMMIT_TOKEN_LATENCY"))
xact_no_backoff = int(workloads.get_var(env_dict, "XACT_NO_BACKOFF"))
enable_magic_waiting = int(workloads.get_var(env_dict, "ENABLE_MAGIC_WAITING"))
xact_enable_virtualization_logtm_se = int(workloads.get_var(env_dict, "XACT_ENABLE_VIRTUALIZATION_LOGTM_SE"))
enable_watchpoint = int(workloads.get_var(env_dict, "ENABLE_WATCHPOINT"))
xact_cr = workloads.get_var(env_dict, "XACT_CONFLICT_RES")
xact_log_buffer_size = int(workloads.get_var(env_dict, "XACT_LOG_BUFFER_SIZE"))
xact_store_predictor_entries = int(workloads.get_var(env_dict, "XACT_STORE_PREDICTOR_ENTRIES"))
xact_store_predictor_history = int(workloads.get_var(env_dict, "XACT_STORE_PREDICTOR_HISTORY"))
xact_store_predictor_threshold = int(workloads.get_var(env_dict, "XACT_STORE_PREDICTOR_THRESHOLD"))
xact_first_access_cost = int(workloads.get_var(env_dict, "XACT_FIRST_ACCESS_COST"))
xact_first_page_access_cost = int(workloads.get_var(env_dict, "XACT_FIRST_PAGE_ACCESS_COST"))
#random_seed = 0
print "condor_cluster: %d" % condor_cluster
print "condor_process: %d" % condor_process
print "random_seed: %d" % random_seed
###### set simics parameters
# enable/disable STCs here
mfacet.run_sim_command("istc-disable")
mfacet.run_sim_command("dstc-disable")
mfacet.run_sim_command("stc-status")
# disable breakpoint before we sync all processors
mfacet.run_sim_command('magic-break-disable')
# always use 1 for better parallelism
old_switch_time = conf.sim.cpu_switch_time
print "Old cpu_switch_time: %d" % conf.sim.cpu_switch_time
mfacet.run_sim_command("cpu-switch-time")
conf.sim.cpu_switch_time = 1
if (SIM_number_processors() == 1):
delay_time = 1
else:
delay_time = old_switch_time
mfacet.run_sim_command("c %d"%(delay_time)) # sync CPUs after changing switch time
mfacet.run_sim_command("cpu-switch-time")
print "New cpu_switch_time: %d" % conf.sim.cpu_switch_time
# enable breakpoint
mfacet.run_sim_command('magic-break-enable')
###### Conserve memory by limiting how much memory each simics image object can use
#mfacet.limit_all_images(256)
# enable IFETCH, i-fetch line size is controlled in checkpoint files
print "Old instruction-fetch-mode: "
mfacet.run_sim_command("instruction-fetch-mode")
mfacet.run_sim_command("instruction-fetch-mode instruction-fetch-trace")
print "New instruction-fetch-mode: "
mfacet.run_sim_command("instruction-fetch-mode")
###### Load Ruby and set parameters
mfacet.run_sim_command("load-module ruby")
mfacet.run_sim_command("ruby0.setparam_str REMOVE_SINGLE_CYCLE_DCACHE_FAST_PATH true")
#mfacet.run_sim_command("ruby0.setparam_str PROFILE_EXCEPTIONS true")
mfacet.run_sim_command("ruby0.setparam_str PROFILE_XACT false")
mfacet.run_sim_command("ruby0.setparam_str PROFILE_NONXACT false")
mfacet.run_sim_command("ruby0.setparam_str XACT_MEMORY true")
mfacet.run_sim_command("ruby0.setparam g_DEADLOCK_THRESHOLD 20000000")
mfacet.run_sim_command("ruby0.setparam g_MEMORY_SIZE_BYTES 8589934592")
mfacet.run_sim_command("ruby0.setparam g_RANDOM_SEED %d" % random_seed)
mfacet.run_sim_command("ruby0.setparam g_PAGE_SIZE_BYTES %d" % 8192)
mfacet.run_sim_command("ruby0.setparam_str XACT_DEBUG false")
mfacet.run_sim_command("ruby0.setparam XACT_DEBUG_LEVEL %d" % 2)
# Need to set this for CMPs!
mfacet.run_sim_command("ruby0.setparam g_PROCS_PER_CHIP %d" % processors_per_chip)
# We don't need to set g_NUM_PROCESSORS explicitly, but need to set # of SMT threads if we want threads per proc> 1
mfacet.run_sim_command("ruby0.setparam g_NUM_SMT_THREADS %d" % smt_threads)
# Number of outstanding requests from each sequencer object (default is 16)
mfacet.run_sim_command("ruby0.setparam g_SEQUENCER_OUTSTANDING_REQUESTS %d" % (16*smt_threads))
# Comment this out for production/faster runs
mfacet.run_sim_command("ruby0.setparam_str XACT_ISOLATION_CHECK true")
# set multipliers here
mfacet.run_sim_command("ruby0.setparam SIMICS_RUBY_MULTIPLIER %d" % 1)
mfacet.run_sim_command("ruby0.setparam OPAL_RUBY_MULTIPLIER %d" % 1)
# 32 kB 4-way L1 cache
mfacet.run_sim_command("ruby0.setparam L1_CACHE_ASSOC %d" % 4)
mfacet.run_sim_command("ruby0.setparam L1_CACHE_NUM_SETS_BITS %d" % 7)
mfacet.run_sim_command("ruby0.setparam SEQUENCER_TO_CONTROLLER_LATENCY %d" % 1) # L1 hit time
# 8 MB 8-way L2 cache
mfacet.run_sim_command("ruby0.setparam L2_CACHE_ASSOC %d" % 8)
mfacet.run_sim_command("ruby0.setparam L2_CACHE_NUM_SETS_BITS %d" % 14)
mfacet.run_sim_command("ruby0.setparam L2_RESPONSE_LATENCY %d" % 20)
mfacet.run_sim_command("ruby0.setparam L2_TAG_LATENCY %d" % 6)
mfacet.run_sim_command("ruby0.setparam L2_REQUEST_LATENCY %d" % 15)
# Main memory
mfacet.run_sim_command("ruby0.setparam MEMORY_RESPONSE_LATENCY_MINUS_2 %d" % 448)
# interconnection network parameters
mfacet.run_sim_command("ruby0.setparam NETWORK_LINK_LATENCY %d" % 14)
# network topology
# use PT_TO_PT for CMPs
# CMPs use Dancehall network topology
mfacet.run_sim_command("ruby0.setparam_str g_NETWORK_TOPOLOGY FILE_SPECIFIED")
#mfacet.run_sim_command("ruby0.setparam_str g_NETWORK_TOPOLOGY PT_TO_PT")
mfacet.run_sim_command("ruby0.setparam RECYCLE_LATENCY %d" % 1)
mfacet.run_sim_command("ruby0.setparam NUMBER_OF_VIRTUAL_NETWORKS %d" % 5)
#mfacet.run_sim_command("ruby0.setparam_str g_PRINT_TOPOLOGY true")
if (processors_per_chip == 16):
mfacet.run_sim_command("ruby0.setparam g_NUM_MEMORIES %d" % 8)
elif (processors_per_chip == 32):
mfacet.run_sim_command("ruby0.setparam g_NUM_MEMORIES %d" % 16)
else:
mfacet.run_sim_command("ruby0.setparam g_NUM_MEMORIES %d" % 8)
mfacet.run_sim_command("ruby0.setparam L2CACHE_TRANSITIONS_PER_RUBY_CYCLE %d" % 1000)
mfacet.run_sim_command("ruby0.setparam DIRECTORY_TRANSITIONS_PER_RUBY_CYCLE %d" % 1000)
#mfacet.run_sim_command("ruby0.setparam_str g_NETWORK_TOPOLOGY HIERARCHICAL_SWITCH")
mfacet.run_sim_command("ruby0.setparam XACT_LOG_BUFFER_SIZE %d" % xact_log_buffer_size)
## Read/Write Filter options
# Physical signatures
filter_type = read_write_filter.split('_')
if(filter_type[0] == "Perfect"):
mfacet.run_sim_command("ruby0.setparam_str PERFECT_FILTER true")
else:
mfacet.run_sim_command("ruby0.setparam_str PERFECT_FILTER false")
# set read/write filter type
mfacet.run_sim_command("ruby0.setparam_str READ_WRITE_FILTER %s" % read_write_filter)
# Virtual signatures
filter_type = virtual_signature.split('_')
if(filter_type[0] == "Perfect"):
mfacet.run_sim_command("ruby0.setparam_str PERFECT_VIRTUAL_FILTER true")
else:
mfacet.run_sim_command("ruby0.setparam_str PERFECT_VIRTUAL_FILTER false")
# set read/write filter type
mfacet.run_sim_command("ruby0.setparam_str VIRTUAL_READ_WRITE_FILTER %s" % virtual_signature)
# Summary signatures
filter_type = summary_signature.split('_')
if(filter_type[0] == "Perfect"):
mfacet.run_sim_command("ruby0.setparam_str PERFECT_SUMMARY_FILTER true")
else:
mfacet.run_sim_command("ruby0.setparam_str PERFECT_SUMMARY_FILTER false")
# set read/write filter type
mfacet.run_sim_command("ruby0.setparam_str SUMMARY_READ_WRITE_FILTER %s" % summary_signature)
mfacet.run_sim_command("ruby0.setparam_str XACT_CONFLICT_RES %s" % xact_cr)
mfacet.run_sim_command("ruby0.setparam XACT_STORE_PREDICTOR_ENTRIES %d" % xact_store_predictor_entries)
mfacet.run_sim_command("ruby0.setparam XACT_STORE_PREDICTOR_HISTORY %d" % xact_store_predictor_history)
mfacet.run_sim_command("ruby0.setparam XACT_STORE_PREDICTOR_THRESHOLD %d" % xact_store_predictor_threshold)
if (xact_visualizer):
mfacet.run_sim_command("ruby0.setparam_str XACT_VISUALIZER true")
if (xact_lazy):
mfacet.run_sim_command("ruby0.setparam_str XACT_LAZY_VM true")
if (not xact_eager):
mfacet.run_sim_command("ruby0.setparam_str XACT_EAGER_CD false")
#enable_tourmaline = 1
if (enable_tourmaline):
mfacet.run_sim_command("ruby0.setparam_str XACT_ENABLE_TOURMALINE true")
if (xact_commit_token > 0):
mfacet.run_sim_command("ruby0.setparam XACT_COMMIT_TOKEN_LATENCY %d" % xact_commit_token)
if (xact_no_backoff):
mfacet.run_sim_command("ruby0.setparam_str XACT_NO_BACKOFF true")
if (enable_magic_waiting):
mfacet.run_sim_command("ruby0.setparam_str ENABLE_MAGIC_WAITING true")
if (xact_enable_virtualization_logtm_se):
mfacet.run_sim_command("ruby0.setparam_str XACT_ENABLE_VIRTUALIZATION_LOGTM_SE true")
if (enable_watchpoint):
mfacet.run_sim_command("ruby0.setparam_str ENABLE_WATCHPOINT true")
if (xact_first_access_cost > 0):
mfacet.run_sim_command("ruby0.setparam XACT_FIRST_ACCESS_COST %d" % xact_first_access_cost)
if (xact_first_page_access_cost > 0):
mfacet.run_sim_command("ruby0.setparam XACT_FIRST_PAGE_ACCESS_COST %d" % xact_first_page_access_cost)
mfacet.run_sim_command("ruby0.init")
#debug = 2820483
if debug > 0:
#mfacet.run_sim_command('ruby0.debug-verb med')
mfacet.run_sim_command('ruby0.debug-verb high')
mfacet.run_sim_command('ruby0.debug-filter lsNqST')
mfacet.run_sim_command('ruby0.debug-start-time "%d"' % debug)
mfacet.run_sim_command('ruby0.debug-output-file /scratch/ruby')
def run_count_test(exe_name, iterations):
os.environ["WORKLOAD"] = exe_name
os.environ["MODULE"] = "MOESI_xact_hammer"
os.environ["BANDWIDTH"] = "10000"
os.environ["CHECKPOINT"] = "golden"
os.environ["RESULTS_DIR"] = "."
os.environ["TRANSACTIONS"] = "0"
if exe_name == "count_locks":
print "USING LOCK-BASED SYNCHRONIZATION"
else:
print "USING TRANSACTIONAL MEMORY"
###### Read simulation parameters
env_dict = workloads.prepare_env_dictionary(simics = 1)
processors = int(workloads.get_var(env_dict, "PROCESSORS"))
#workloads.print_all_variables(env_dict)
checkpoint = "golden"
ruby = 1 # true
opal = None
###### print out local host name to help with troubleshooting
print "Local host name:", string.strip(os.popen("hostname").readline())
###### init simics with a checkpoint
if checkpoint[0] == '/':
mfacet.run_sim_command('read-configuration "%s"' % (checkpoint))
else:
mfacet.run_sim_command('read-configuration "../../checkpoints-u3/%s-%dp.check"' % (checkpoint, processors));
#mfacet.run_sim_command("c 100000")
mfacet.run_sim_command('magic-break-enable')
lines = ["bash\n",
"mount /host\n",
"cp /host/s/gcc-3.3/sun4x_58/lib/libstdc++.so.5 .\n",
"cp /host/s/gcc-3.3/sun4x_58/lib/libgcc_s.so.1 .\n",
#"cp /host/p/multifacet/projects/xact_memory/multifacet/microbenchmarks/transactional/dining_philosophers/dphil .\n",
#"cp /host/p/multifacet/projects/xact_memory/multifacet/microbenchmarks/transactional/linked-list/list_bench .\n",
#"cp /host/p/multifacet/projects/xact_memory/multifacet/microbenchmarks/transactional/shared-counter/count .\n",
"cp /host/p/multifacet/projects/java_locks/nuca_locks/src/libparmacs_locks.so .\n",
"cp /host/p/multifacet/projects/java_locks/nuca_locks/nuca_bench/%s .\n" % exe_name,
"cp /host/p/multifacet/projects/java_locks/nuca_locks/nuca_bench/WILDFIRE.on .\n",
"export LD_LIBRARY_PATH=.\n",
"source WILDFIRE.on\n",
]
proc_no = [0, 1, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
for j in range(processors):
if(j != 0):
print "j=%d, num_p=%d" % (j, processors)
lines.append("psrset -c\n")
print "psrset -c\n"
lines.append("psrset -a %d %d\n" % (j, proc_no[j]))
print "psrset -a %d %d\n" % (j, proc_no[j])
#for i in lines:
#insert_cmd(SIM_get_object("con0"), i)
# break_cmd(SIM_get_object("con0"), "bash-2.05#")
# mfacet.run_sim_command("c")
# unbreak_cmd(SIM_get_object("con0"), "bash-2.05#")
lines.append("./%s %d %d\n" % (exe_name, processors, iterations))
mfacet.console_commands(lines, "bash-2.05#")
#insert_cmd(SIM_get_object("con0"), "ls -l\n")
#insert_cmd(SIM_get_object("con0"), "ls -l\n")
#insert_cmd(SIM_get_object("con0"), "./dphil 10\n")
#insert_cmd(SIM_get_object("con0"), "./list_bench %d\n" % processors)
#insert_cmd(SIM_get_object("con0"), "./%s %d %d; magic_call break\n" % (exe_name, processors, iterations))
#insert_cmd(SIM_get_object("con0"), "./%s %d %d\n" % (exe_name, processors, iterations))
mfacet.run_sim_command("c") # run up to magic_break
mfacet.run_sim_command("c 1") # run up to magic_break
def start_ruby_small_cache():
env_dict = workloads.prepare_env_dictionary(simics = 1)
random_seed = workloads.get_var(env_dict, "RANDOM_SEED")
#condor_cluster = int(workloads.get_var(env_dict, "CONDORCLUSTER"))
#condor_process = int(workloads.get_var(env_dict, "CONDORPROCESS"))
#if random_seed == None:
# random_seed = (condor_cluster << 16) | condor_process
#else:
# random_seed = int(random_seed)
random_seed = 0
###### set simics parameters
# enable/disable STCs here
mfacet.run_sim_command("istc-disable")
mfacet.run_sim_command("dstc-disable")
mfacet.run_sim_command("stc-status")
# disable breakpoint before we sync all processors
mfacet.run_sim_command('magic-break-disable')
# always use 1 for better parallelism
old_switch_time = conf.sim.cpu_switch_time
print "Old cpu_switch_time: %d" % conf.sim.cpu_switch_time
conf.sim.cpu_switch_time = 1
#if (SIM_number_processors() == 1):
# delay_time = 1
#else:
# delay_time = old_switch_time
#mfacet.run_sim_command("c %d"%(delay_time)) # sync CPUs after changing switch time
#print "New cpu_switch_time: %d" % conf.sim.cpu_switch_time
# enable breakpoint
mfacet.run_sim_command('magic-break-enable')
###### Conserve memory by limiting how much memory each simics image object can use
#mfacet.limit_all_images(256)
# enable IFETCH, i-fetch line size is controlled in checkpoint files
#mfacet.run_sim_command("instruction-profile-mode instruction-cache-access-trace")
#mfacet.run_sim_command("instruction-profile-mode")
#import whrandom
#random_seed = whrandom.randint(0,100000)
###### Load Ruby and set parameters
mfacet.run_sim_command("load-module ruby")
#mfacet.run_sim_command("ruby0.set-seed(%d)" % random_seed)
#mfacet.run_sim_command("ruby0.fix-bandwidth(%d)" % 64000)
#mfacet.run_sim_command("ruby0.setparam PROCS_PER_CHIP %d" % 2)
#mfacet.run_sim_command("ruby0.setparam_str g_NETWORK_TOPOLOGY CMP_DIRECT_SWITCH")
#mfacet.run_sim_command("ruby0.setparam MEMORY_LATENCY 1")
#mfacet.run_sim_command("ruby0.setparam NETWORK_LINK_LATENCY 1")
#mfacet.run_sim_command("ruby0.setparam DIRECTORY_LATENCY 1")
mfacet.run_sim_command("ruby0.setparam g_DEADLOCK_THRESHOLD 2000000")
mfacet.run_sim_command("ruby0.setparam g_MEMORY_SIZE_BYTES 8589934592")
mfacet.run_sim_command("ruby0.setparam_str REMOVE_SINGLE_CYCLE_DCACHE_FAST_PATH true")
mfacet.run_sim_command("ruby0.setparam g_RANDOM_SEED %d" % random_seed)
#mfacet.run_sim_command("ruby0.setparam RETRY_LATENCY %d" % 5)
mfacet.run_sim_command("ruby0.setparam L1_CACHE_ASSOC %d" % 1)
mfacet.run_sim_command("ruby0.setparam L1_CACHE_NUM_SETS_BITS %d" % 4)
mfacet.run_sim_command("ruby0.setparam L2_CACHE_ASSOC %d" % 1)
mfacet.run_sim_command("ruby0.setparam L2_CACHE_NUM_SETS_BITS %d" % 6)
mfacet.run_sim_command("ruby0.init")
def run_simics(checkpoint, workload_name,
transactions,
protocol={"name" : "test"},
processors=16,
smt_threads=1,
phy_signature="Perfect_",
virtual_signature="Perfect_",
summary_signature="Perfect_",
xact_max_depth="1",
microbenchmark = 0,
mbench_arg_prefix="",
mbench_arg_string="",
bench_arg = 0,
condor_process = os.getpid(), condor_cluster = os.getppid(),
expected_ruby_cycles=0, tolerance=.05,
check_opal=0, check_ruby=0,
):
# use SPARC V8 directory
name = protocol["name"]
output = tools.run_command("scripts/prepare_simics_home.sh simics/home/%s"%name)
if protocol.has_key("specific_processor_count"):
processors = protocol.get("specific_processor_count", 1)
procs_per_chip = protocol.get("procs_per_chip", 1)
l2_cache_banks = protocol.get("specific_cache_count", (processors/smt_threads))
eager_cd = protocol.get("eager_cd", 1)
eager_vm = protocol.get("eager_vm", 1)
magic_waiting = protocol.get("magic_waiting", 0)
no_backoff = protocol.get("no_backoff", 0)
bandwidth = protocol.get("bandwidth", 6400)
if procs_per_chip > processors:
procs_per_chip = 1
log_message("Running simics: checkpoint=%s, processors=%s, smt_threads=%s, procs_per_chip=%d transactions=%d, protocol: %s" % (checkpoint, processors, smt_threads, procs_per_chip, transactions, name))
# prepare environment variables for running simics
env_dict = workloads.prepare_env_dictionary(simics = 0)
workloads.set_var(env_dict, "RESULTS_DIR", "../../../results")
workloads.set_var(env_dict, "WORKLOAD", workload_name)
workloads.set_var(env_dict, "CHECKPOINT", checkpoint)
workloads.set_var(env_dict, "CHECKPOINT_DIR", " ")
workloads.set_var(env_dict, "PROTOCOL", name)
workloads.set_var(env_dict, "PROCESSORS", processors)
workloads.set_var(env_dict, "CHIPS", 1)
workloads.set_var(env_dict, "SMT_THREADS", smt_threads)
workloads.set_var(env_dict, "PROCS_PER_CHIP", procs_per_chip)
workloads.set_var(env_dict, "NUM_L2_BANKS", l2_cache_banks)
workloads.set_var(env_dict, "TRANSACTIONS", transactions)
workloads.set_var(env_dict, "BANDWIDTH", bandwidth)
if(g_random_seed != "random"):
workloads.set_var(env_dict, "RANDOM_SEED", g_random_seed)
workloads.set_var(env_dict, "CONDORCLUSTER", condor_cluster)
workloads.set_var(env_dict, "CONDORPROCESS", condor_process)
# Transactional Memory variables
workloads.set_var(env_dict, "MICROBENCH_DIR", "microbenchmarks/transactional")
workloads.set_var(env_dict, "BENCHMARK", workload_name)
workloads.set_var(env_dict, "MAX_DEPTH", xact_max_depth)
workloads.set_var(env_dict, 'LOCK_TYPE', "TM")
workloads.set_var(env_dict, 'READ_WRITE_FILTER', phy_signature)
workloads.set_var(env_dict, 'VIRTUAL_READ_WRITE_FILTER', virtual_signature)
workloads.set_var(env_dict, 'SUMMARY_READ_WRITE_FILTER', summary_signature)
workloads.set_var(env_dict, "XACT_EAGER_CD", eager_cd)
if(eager_vm == 0):
workloads.set_var(env_dict, "XACT_LAZY_VM", 1)
else:
workloads.set_var(env_dict, "XACT_LAZY_VM", 0)
workloads.set_var(env_dict, "ENABLE_MAGIC_WAITING", magic_waiting)
workloads.set_var(env_dict, "XACT_NO_BACKOFF", no_backoff)
# set per-microbenchmark specific variables
#if ((workload_name == "compensation") or (workload_name == "commit-action")
# or (workload_name == "isolation-test") or (workload_name == "logging-test")
# or (workload_name == "partial-rollback")):
# workloads.set_var(env_dict, "MICROBENCH_DIR", "microbenchmarks/transactional/test")
workloads.set_var(env_dict, 'MBENCH_ARG_PREFIX', mbench_arg_prefix)
workloads.set_var(env_dict, 'MBENCH_ARG_STRING', mbench_arg_string)
workloads.update_system_env(env_dict)
# create results directory
output = tools.run_command("/bin/rm -rf results")
mbench_arg_prefix = workloads.get_var(env_dict, 'MBENCH_ARG_PREFIX')
os.mkdir("results")
if(microbenchmark == 1):
print "CREATING DIRECTORY results/%s-%s" % (workload_name,mbench_arg_prefix)
os.mkdir("results/%s-%s" %(workload_name,mbench_arg_prefix))
else:
os.mkdir("results/"+workload_name)
print "WORKLOAD NAME %s" % workload_name
#'''
os.chdir("simics/home/%s/" % name)
# run the microbenchmark script if needed
if( microbenchmark == 1):
output = tools.run_command("./simics -echo -verbose -no-log -no-win -x ../../../gen-scripts/microbench.simics", "quit 666\n", verbose=1, max_lines=0)
else:
output = tools.run_command("./simics -echo -verbose -no-log -no-win -x ../../../gen-scripts/go.simics", "quit 666\n", verbose=1)
#tools.run_command("./simics -echo -verbose -no-log -x ../../../gen-scripts/go.simics", "quit 666\n")
os.chdir("../../..")
# dump simics output
if(microbenchmark == 0):
simics_output_filename = "results/%s.output" % workloads.get_output_file_name_prefix(env_dict)
else:
simics_output_filename = "results/%s.output" % workloads.get_microbench_output_file_name_prefix(env_dict,0)
simics_output = open(simics_output_filename, "w")
simics_output.write(output)
simics_output.close()
if check_ruby == 1 and name != "template":
if( microbenchmark == 0):
ruby_stats_filename = "results/%s.stats" % workloads.get_output_file_name_prefix(env_dict)
error_output_filename = "condor/results/%s.error" % (workloads.get_output_file_name_prefix(env_dict, 0))
else:
ruby_stats_filename = "results/%s.stats" % workloads.get_microbench_output_file_name_prefix(env_dict,0)
error_output_filename = "condor/results/%s.error" % (workloads.get_microbench_output_file_name_prefix(env_dict, 0))
if (not os.path.exists(ruby_stats_filename)):
raise RegressionError("Ruby stats output file not present: %s" % ruby_stats_filename, output)
# Check for error file, indicating a SIMICS_ASSERT() failure
if(os.path.exists(error_output_filename)):
print "SIMICS ASSERT error!"
raise RegressionError("SIMICS_ASSERT error file found: %s" % error_output_filename, output)
# get random seed
simics_output = open(simics_output_filename, "r")
for line in simics_output.readlines():
if re.search("g_RANDOM_SEED", line):
tokens = line.split()
log_message(" Random seed: %d"%int(tokens[4][:-1]))
# get ruby cycle
ruby_stats = open(ruby_stats_filename, "r")
ruby_cycles = 0
for line in ruby_stats.readlines():
line_elements = string.split(line)
if len(line_elements) > 1 and line_elements[0] == "Ruby_cycles:":
ruby_cycles = int(line_elements[1])
if (ruby_cycles == 0):
raise RegressionError("Ruby_cycles not found from the output file: %s" % ruby_stats_filename, output)
else:
log_message(" Ruby_cycles: %d"%ruby_cycles)
if (expected_ruby_cycles != 0):
percent_diff = 1.0*ruby_cycles/expected_ruby_cycles
if percent_diff < (1.0-tolerance) or percent_diff > (1.0 + tolerance):
log_message(" Checking ruby_cycles - ratio is %f: OUT OF RANGE" % percent_diff)
log_error("ERROR: Ruby_cycles not within tolerances. expected %d, actual %d" % (expected_ruby_cycles, ruby_cycles))
else:
log_message(" Checking ruby_cycles - ratio is %f: OK" % percent_diff)
if check_opal == 1:
opal_log_filename = "results/%s.opal" % workloads.get_output_file_name_prefix(env_dict)
if (not os.path.exists(opal_log_filename)):
raise RegressionError(("Opal log file not present: %s" %
opal_log_filename), output)
# check opal correct rate!
else:
opal_log = open(opal_log_filename)
processor_total_instructions = 1001 # > 1000
processor_correct_rate = 98 # < 99
for line in opal_log.readlines():
tokens = line.split()
# remember the correct rate
if(len(tokens) == 5 and tokens[1] == "Percent" and tokens[2] == "correct"):
processor_correct_rate = float(tokens[4])
# remember the processor's commit instruction number
if(len(tokens) == 6 and tokens[1] == "Total" and tokens[2] == "number" and tokens[3] == "of" and tokens [4] == "instructions"):
processor_total_instructions = int(tokens[5])
# check the correct rate here since the total instruction
# number comes last during the scan of the output file
if(processor_correct_rate < 99 and processor_total_instructions > 1000):
raise RegressionError(("Opal correct rate too low (%f%% of %d instructions)!" % (processor_correct_rate, processor_total_instructions)), output)
def load_modules_and_run(fast_forward=0):
print "Python version: %s" % sys.version
###### Read simulation parameters
env_dict = workloads.prepare_env_dictionary(simics=1)
workloads.check_requirements(env_dict)
workloads.print_all_variables(env_dict)
protocol = workloads.get_var(env_dict, "PROTOCOL")
workload_name = workloads.get_var(env_dict, "WORKLOAD")
checkpoint_dir = workloads.get_var(env_dict, "CHECKPOINT_DIR")
checkpoint = workloads.get_var(env_dict, "CHECKPOINT")
chips = int(workloads.get_var(env_dict, "CHIPS"))
processors_per_chip = int(workloads.get_var(env_dict, "PROCS_PER_CHIP"))
smt_threads = int(workloads.get_var(env_dict, "SMT_THREADS"))
l2_banks = int(workloads.get_var(env_dict, "NUM_L2_BANKS"))
bandwidth = int(workloads.get_var(env_dict, "BANDWIDTH"))
results_dir = workloads.get_var(env_dict, "RESULTS_DIR")
transactions = int(workloads.get_var(env_dict, "TRANSACTIONS"))
dump_interval = int(workloads.get_var(env_dict, "DUMP_INTERVAL"))
condor_cluster = int(workloads.get_var(env_dict, "CONDORCLUSTER"))
condor_process = int(workloads.get_var(env_dict, "CONDORPROCESS"))
checkpoint_at_end = workloads.get_var(env_dict, "CHECKPOINT_AT_END")
generate_request_trace = workloads.get_var(env_dict, "GENERATE_TRACE")
generate_cache_data_dump = workloads.get_var(env_dict, "CACHE_DATA_DUMP")
protocol_option = workloads.get_var(env_dict, "PROTOCOL_OPTION")
network_topology = workloads.get_var(env_dict, "NETWORK_TOPOLOGY")
warmup_file = workloads.get_var(env_dict, "WARMUP_FILE")
opal_config_file = workloads.get_var(env_dict, "OPAL_CONFIG_FILE")
opal_config_name = workloads.get_var(env_dict, "OPAL_CONFIG_NAME")
interactive = workloads.get_var(env_dict, "INTERACTIVE")
random_seed = workloads.get_var(env_dict, "RANDOM_SEED")
use_local_mirror = workloads.get_var(env_dict, "USE_LOCAL_MIRROR")
if random_seed == None:
random_seed = (condor_cluster << 16) | condor_process
else:
random_seed = int(random_seed)
###### mandatory parameters
assert (protocol != None)
assert (workload_name != None)
assert (checkpoint != None)
assert (chips != None)
assert (processors_per_chip != None)
assert (smt_threads != None)
assert (results_dir != None)
assert (transactions != None)
assert (dump_interval != None)
assert (condor_cluster != None)
assert (condor_process != None)
ruby = None
if "ruby" in get_module_list():
ruby = 1 # true
opal = None
if "opal" in get_module_list():
opal = 1 # true
###### print out local host name to help with troubleshooting
print "Local host name:", string.strip(os.popen("hostname").readline())
###### init simics with a checkpoint
assert (checkpoint[0] == '/')
run_sim_command('read-configuration "%s/%s"' %
(checkpoint_dir, checkpoint))
if (fast_forward): run_sim_command("continue %d" % fast_forward)
###### Conserve memory by limiting how much memory each simics image object can use
#@mfacet.limit_all_images(256)
###### set simics parameters
# enable/disable STCs here
run_sim_command("istc-disable")
run_sim_command("dstc-disable")
run_sim_command("stc-status")
# disable breakpoint before we sync all processors
run_sim_command('magic-break-disable')
# always use 1 for better parallelism
old_switch_time = conf.sim.cpu_switch_time
print "Old cpu_switch_time: %d" % conf.sim.cpu_switch_time
conf.sim.cpu_switch_time = 1
# 8/28/2003 CM: stepping or continuing simics before loading opal results
# in a major performance loss. See me for more details.
if not opal:
run_sim_command(
"c %d" %
(10 * old_switch_time)) # sync CPUs after changing switch time
print "New cpu_switch_time: %d" % conf.sim.cpu_switch_time
# enable breakpoint
run_sim_command('magic-break-enable')
# enable IFETCH, i-fetch line size is controlled in checkpoint files
run_sim_command("instruction-fetch-mode instruction-fetch-trace")
run_sim_command("instruction-fetch-mode")
filename_prefix = "%s/%s" % (
results_dir, workloads.get_output_file_name_prefix(env_dict, 1))
prep_dir(filename_prefix)
###### capture xterm output in file
import mod_xterm_console_commands
mod_xterm_console_commands.cap_start_cmd(get_console(),
filename_prefix + ".xterm")
###### Load modules and set parameters
if opal:
run_sim_command('load-module opal')
if ruby:
run_sim_command("load-module ruby")
if network_topology:
run_sim_command('ruby0.setparam_str g_NETWORK_TOPOLOGY "%s"' %
network_topology)
if protocol_option:
run_sim_command('ruby0.setparam_str g_MASK_PREDICTOR_CONFIG "%s"' %
protocol_option)
run_sim_command("ruby0.setparam g_PROCS_PER_CHIP %d" %
processors_per_chip)
run_sim_command("ruby0.setparam g_NUM_L2_BANKS %d" % l2_banks)
run_sim_command("ruby0.setparam g_RANDOM_SEED %d" % random_seed)
run_sim_command("ruby0.setparam g_endpoint_bandwidth %d" % bandwidth)
run_sim_command("ruby0.setparam g_NUM_PROCESSORS %d" %
(chips * processors_per_chip))
run_sim_command("ruby0.setparam g_NUM_SMT_THREADS %d" % smt_threads)
transactional_memory = 0
if transactional_memory:
run_sim_command(
"ruby0.setparam_str REMOVE_SINGLE_CYCLE_DCACHE_FAST_PATH true")
run_sim_command("ruby0.setparam_str PROFILE_EXCEPTIONS true")
run_sim_command("ruby0.setparam_str PROFILE_XACT true")
run_sim_command("ruby0.setparam_str XACT_MEMORY true")
run_sim_command("ruby0.setparam XACT_MAX_DEPTH %s" %
xact_max_depth)
#run_sim_command("ruby0.setparam_str DATA_BLOCK true")
run_sim_command("ruby0.setparam RETRY_LATENCY 10")
#run_sim_command("ruby0.setparam RANDOM_SEED 0")
run_sim_command("ruby0.setparam g_MEMORY_SIZE_BYTES 8589934592")
run_sim_command("ruby0.setparam g_DEADLOCK_THRESHOLD 20000000")
run_sim_command("ruby0.init")
run_sim_command("ruby0.periodic-stats-interval %d" % 1000000)
run_sim_command('ruby0.periodic-stats-file "%s.periodic" ' %
filename_prefix)
debug = 0
if debug:
run_sim_command('ruby0.debug-verb high')
run_sim_command('ruby0.debug-filter lseagTSN')
#run_sim_command('ruby0.debug-start-time "500000"')
run_sim_command('ruby0.debug-start-time "0"')
###### init opal
if opal:
# For debugging with symtable, used by Min's race detector
#run_sim_command('load-module symtable')
#run_sim_command('new-symtable mysql')
# set up the number of SMT Threads per physical processor::
run_sim_command("opal0.setparam CONFIG_LOGICAL_PER_PHY_PROC %d" %
smt_threads)
# set up the number of physical registers needed, for each physical register type:
run_sim_command("opal0.setparam CONFIG_IREG_PHYSICAL %d" %
(160 * smt_threads + 64))
run_sim_command("opal0.setparam CONFIG_FPREG_PHYSICAL %d" %
(64 * smt_threads + 128))
run_sim_command("opal0.setparam CONFIG_CCREG_PHYSICAL %d" %
(5 * smt_threads + 64))
if opal_config_file:
run_sim_command('opal0.init "%s"' % opal_config_file)
else:
run_sim_command('opal0.init')
run_sim_command('opal0.sim-start "%s.opal"' % filename_prefix)
###### do the following only when it is not interactive simulation
if not interactive:
###### warm up the caches
if ruby and warmup_file:
print "Warming caches with %s" % warmup_file
run_sim_command('ruby0.load-caches "%s/%s"' %
(checkpoint_dir, warmup_file))
###### generate request trace
if ruby and generate_request_trace:
trace_filename = "/scratch/%s/traces/%s-%dc-%dp-%dt-%s-%d-%d-%d.trace.gz" % (
getpass.getuser(), workload_name, chips, processors_per_chip,
smt_threads, protocol, bandwidth, condor_cluster,
condor_process)
prep_dir(trace_filename)
run_sim_command("ruby0.tracer-output-file %s" % trace_filename)
###### Run for n transactions or just a short run
print "Initial memory usage: %s" % memory_usage_str()
print "Running..."
if transactions > 0:
assert (dump_interval > 0)
setup_run_for_n_transactions(transactions, dump_interval)
if opal:
run_sim_command("opal0.sim-step 100000000000")
else:
run_sim_command("c")
else:
if opal:
run_sim_command("opal0.sim-step 1000")
else:
run_sim_command("c 10000")
###### dump statistics
if ruby:
run_sim_command('ruby0.dump-stats "%s.stats"' % filename_prefix)
if generate_cache_data_dump:
run_sim_command('ruby0.dump-cache-data 0 "%s.cache"' %
filename_prefix)
if opal:
run_sim_command('opal0.listparam')
run_sim_command('opal0.stats')
run_sim_command('opal0.sim-stop')
if checkpoint_at_end == 'yes':
mf_write_configuration_cmd(
"/scratch/%s/%s-%dc-%dp-%dt-%d-caches" %
(getpass.getuser(), checkpoint, chips, processors_per_chip,
smt_threads, transactions))
# close trace file
if ruby and generate_request_trace == 'yes':
run_sim_command('ruby0.tracer-output-file ""')
###### display resource usage
print_resource_usage()
###### close xterm capture file
get_console().output_file = ""
###### finishing up
run_sim_command('quit')
else:
print "enter interactive mode..."
print "Normally, the following command lines will be issued:"
print ""
print " @from mfacet import *"
print " @setup_run_for_n_transactions(%d, %d) % (transactions, dump_interval)"
print ""
print " # (optionally)"
print " ruby0.load-caches <workload>.warmup.gz"
print " ruby0.tracer-output-file <filename>.trace.gz"
print ""
print " # (if ruby only)"
print " continue"
print " ruby0.dump-stats file.stats"
print ""
print " # (if ruby + opal)"
print " opal0.sim-step 100000000000"
print " opal0.listparam"
print " opal0.stats"
print " opal0.sim-stop"
print ""
print " quit"
def end_transaction_callback(desc, cpu, val):
if (val > 0x10000):
val = val >> 16
global __start_transaction_counter
global __end_transaction_counter
if val == 5: # end of trans
#print "end of transaction of thread %d" % get_tid()
__end_transaction_counter += 1
#print_cpu_regs(cpu)
elif val == 3: # start of trans
#print "start of transaction of thread %d" % get_tid()
__start_transaction_counter += 1
elif val == 4: # break simulation
return
elif val >= 6 and val < 15:
return
elif val == 15:
print "ASSERT FAILURE at PC %x" % SIM_get_program_counter(
SIM_current_processor())
# dump a .error file
env_dict = workloads.prepare_env_dictionary(simics=1)
results_dir = workloads.get_var(env_dict, "RESULTS_DIR")
processors = workloads.get_var(env_dict, "PROCESSORS")
print processors
error_output_filename = "%s/%s.error" % (
results_dir,
workloads.get_microbench_output_file_name_prefix(env_dict, 0))
error_output = open(error_output_filename, "w")
error_output.write("SIMICS ASSERT FAILED at PC %x" %
SIM_get_program_counter(SIM_current_processor()))
error_output.close()
SIM_hap_delete_callback("Core_Magic_Instruction",
end_transaction_callback,
"end_transaction_magic_call")
try:
opal0 = SIM_get_object("opal0")
SIM_set_attribute(opal0, "break_simulation", 1)
except:
# if opal if not installed, call simics to halt simulation
SIM_break_simulation("%s: %d transactions reached" %
(desc, __end_transaction_counter))
elif val >= 16 and val <= 62:
return
elif val >= 1024 and val < 7168:
return
else:
print "%s: Unexpected magic call number %d" % (desc, val)
# dump sim_cycle and sim_inst out
if __transaction_dump_interval != 0 and __end_transaction_counter % __transaction_dump_interval == 0:
# dump the transaction count
print "%s: transaction started: %d, transaction completed: %d, transaction_limit: %d, %s" % (
desc, __start_transaction_counter, __end_transaction_counter,
__transaction_limit, memory_usage_str())
# if the trace module is installed, dump statistics
try:
trace0 = SIM_get_object("trace0")
SIM_set_attribute(trace0, "dump_stats", __end_transaction_counter)
except:
pass
if __end_transaction_counter >= __transaction_limit:
print "%s: limit reached, unregistering callback" % desc
SIM_hap_delete_callback("Core_Magic_Instruction",
end_transaction_callback,
"end_transaction_magic_call")
# if opal is installed, tell it to halt simulation
try:
opal0 = SIM_get_object("opal0")
SIM_set_attribute(opal0, "break_simulation", 1)
except:
# if opal if not installed, call simics to halt simulation
SIM_break_simulation("%s: %d transactions reached" %
(desc, __end_transaction_counter))