def load_modules_and_run(fast_forward = 0):
print "Python version: %s" % sys.version
###### Read simulation parameters
env_dict = workloads.prepare_env_dictionary(simics = 1)
workloads.check_requirements(env_dict)
workloads.print_all_variables(env_dict)
protocol = workloads.get_var(env_dict, "PROTOCOL")
workload_name = workloads.get_var(env_dict, "WORKLOAD")
checkpoint_dir = workloads.get_var(env_dict, "CHECKPOINT_DIR")
checkpoint = workloads.get_var(env_dict, "CHECKPOINT")
chips = int(workloads.get_var(env_dict, "CHIPS"))
processors_per_chip = int(workloads.get_var(env_dict, "PROCS_PER_CHIP"))
smt_threads = int(workloads.get_var(env_dict, "SMT_THREADS"))
l2_banks = int(workloads.get_var(env_dict, "NUM_L2_BANKS"))
bandwidth = int(workloads.get_var(env_dict, "BANDWIDTH"))
results_dir = workloads.get_var(env_dict, "RESULTS_DIR")
transactions = int(workloads.get_var(env_dict, "TRANSACTIONS"))
dump_interval = int(workloads.get_var(env_dict, "DUMP_INTERVAL"))
condor_cluster = int(workloads.get_var(env_dict, "CONDORCLUSTER"))
condor_process = int(workloads.get_var(env_dict, "CONDORPROCESS"))
checkpoint_at_end = workloads.get_var(env_dict, "CHECKPOINT_AT_END")
generate_request_trace = workloads.get_var(env_dict, "GENERATE_TRACE")
generate_cache_data_dump = workloads.get_var(env_dict, "CACHE_DATA_DUMP")
protocol_option = workloads.get_var(env_dict, "PROTOCOL_OPTION")
network_topology = workloads.get_var(env_dict, "NETWORK_TOPOLOGY")
warmup_file = workloads.get_var(env_dict, "WARMUP_FILE")
opal_config_file = workloads.get_var(env_dict, "OPAL_CONFIG_FILE")
opal_config_name = workloads.get_var(env_dict, "OPAL_CONFIG_NAME")
interactive = workloads.get_var(env_dict, "INTERACTIVE")
random_seed = workloads.get_var(env_dict, "RANDOM_SEED")
use_local_mirror = workloads.get_var(env_dict, "USE_LOCAL_MIRROR")
if random_seed == None:
random_seed = (condor_cluster << 16) | condor_process
else:
random_seed = int(random_seed)
###### mandatory parameters
assert(protocol != None)
assert(workload_name != None)
assert(checkpoint != None)
assert(chips != None)
assert(processors_per_chip != None)
assert(smt_threads != None)
assert(results_dir != None)
assert(transactions != None)
assert(dump_interval != None)
assert(condor_cluster != None)
assert(condor_process != None)
ruby = None
if "ruby" in get_module_list():
ruby = 1 # true
opal = None
if "opal" in get_module_list():
opal = 1 # true
###### print out local host name to help with troubleshooting
print "Local host name:", string.strip(os.popen("hostname").readline())
###### init simics with a checkpoint
assert(checkpoint[0] == '/')
run_sim_command('read-configuration "%s/%s"' % (checkpoint_dir, checkpoint))
if(fast_forward): run_sim_command("continue %d" % fast_forward)
###### Conserve memory by limiting how much memory each simics image object can use
#@mfacet.limit_all_images(256)
###### set simics parameters
# enable/disable STCs here
run_sim_command("istc-disable")
run_sim_command("dstc-disable")
run_sim_command("stc-status")
# disable breakpoint before we sync all processors
run_sim_command('magic-break-disable')
# always use 1 for better parallelism
old_switch_time = conf.sim.cpu_switch_time
print "Old cpu_switch_time: %d" % conf.sim.cpu_switch_time
conf.sim.cpu_switch_time = 1
# 8/28/2003 CM: stepping or continuing simics before loading opal results
# in a major performance loss. See me for more details.
if not opal:
run_sim_command("c %d"%(10*old_switch_time)) # sync CPUs after changing switch time
print "New cpu_switch_time: %d" % conf.sim.cpu_switch_time
# enable breakpoint
run_sim_command('magic-break-enable')
# enable IFETCH, i-fetch line size is controlled in checkpoint files
run_sim_command("instruction-fetch-mode instruction-fetch-trace")
run_sim_command("instruction-fetch-mode")
filename_prefix = "%s/%s" % (results_dir, workloads.get_output_file_name_prefix(env_dict, 1))
prep_dir(filename_prefix)
###### capture xterm output in file
import mod_xterm_console_commands
mod_xterm_console_commands.cap_start_cmd(get_console(), filename_prefix + ".xterm")
###### Load modules and set parameters
if opal:
run_sim_command('load-module opal')
if ruby:
run_sim_command("load-module ruby")
if network_topology:
run_sim_command('ruby0.setparam_str g_NETWORK_TOPOLOGY "%s"' % network_topology)
if protocol_option:
run_sim_command('ruby0.setparam_str g_MASK_PREDICTOR_CONFIG "%s"' % protocol_option)
run_sim_command("ruby0.setparam g_PROCS_PER_CHIP %d" % processors_per_chip)
run_sim_command("ruby0.setparam g_NUM_L2_BANKS %d" % l2_banks)
run_sim_command("ruby0.setparam g_RANDOM_SEED %d" % random_seed)
run_sim_command("ruby0.setparam g_endpoint_bandwidth %d" % bandwidth)
run_sim_command("ruby0.setparam g_NUM_PROCESSORS %d" % (chips*processors_per_chip))
run_sim_command("ruby0.setparam g_NUM_SMT_THREADS %d" % smt_threads)
transactional_memory = 0
if transactional_memory:
run_sim_command("ruby0.setparam_str REMOVE_SINGLE_CYCLE_DCACHE_FAST_PATH true")
run_sim_command("ruby0.setparam_str PROFILE_EXCEPTIONS true")
run_sim_command("ruby0.setparam_str PROFILE_XACT true")
run_sim_command("ruby0.setparam_str XACT_MEMORY true")
run_sim_command("ruby0.setparam XACT_MAX_DEPTH %s" % xact_max_depth);
#run_sim_command("ruby0.setparam_str DATA_BLOCK true")
run_sim_command("ruby0.setparam RETRY_LATENCY 10")
#run_sim_command("ruby0.setparam RANDOM_SEED 0")
run_sim_command("ruby0.setparam g_MEMORY_SIZE_BYTES 8589934592")
run_sim_command("ruby0.setparam g_DEADLOCK_THRESHOLD 20000000")
run_sim_command("ruby0.init")
run_sim_command("ruby0.periodic-stats-interval %d" % 1000000)
run_sim_command('ruby0.periodic-stats-file "%s.periodic" ' % filename_prefix)
debug = 0
if debug:
run_sim_command('ruby0.debug-verb high')
run_sim_command('ruby0.debug-filter lseagTSN')
#run_sim_command('ruby0.debug-start-time "500000"')
run_sim_command('ruby0.debug-start-time "0"')
###### init opal
if opal:
# For debugging with symtable, used by Min's race detector
#run_sim_command('load-module symtable')
#run_sim_command('new-symtable mysql')
# set up the number of SMT Threads per physical processor::
run_sim_command("opal0.setparam CONFIG_LOGICAL_PER_PHY_PROC %d" % smt_threads)
# set up the number of physical registers needed, for each physical register type:
run_sim_command("opal0.setparam CONFIG_IREG_PHYSICAL %d" % (160*smt_threads+64))
run_sim_command("opal0.setparam CONFIG_FPREG_PHYSICAL %d" % (64*smt_threads+128))
run_sim_command("opal0.setparam CONFIG_CCREG_PHYSICAL %d" % (5*smt_threads+64))
if opal_config_file:
run_sim_command('opal0.init "%s"' % opal_config_file)
else:
run_sim_command('opal0.init')
run_sim_command('opal0.sim-start "%s.opal"' % filename_prefix)
###### do the following only when it is not interactive simulation
if not interactive:
###### warm up the caches
if ruby and warmup_file:
print "Warming caches with %s" % warmup_file
run_sim_command('ruby0.load-caches "%s/%s"' % (checkpoint_dir, warmup_file))
###### generate request trace
if ruby and generate_request_trace:
trace_filename = "/scratch/%s/traces/%s-%dc-%dp-%dt-%s-%d-%d-%d.trace.gz" % (getpass.getuser(), workload_name, chips, processors_per_chip, smt_threads, protocol, bandwidth, condor_cluster, condor_process)
prep_dir(trace_filename)
run_sim_command("ruby0.tracer-output-file %s" % trace_filename)
###### Run for n transactions or just a short run
print "Initial memory usage: %s" % memory_usage_str()
print "Running..."
if transactions > 0:
assert(dump_interval > 0)
setup_run_for_n_transactions(transactions, dump_interval)
if opal:
run_sim_command("opal0.sim-step 100000000000")
else:
run_sim_command("c")
else:
if opal:
run_sim_command("opal0.sim-step 1000")
else:
run_sim_command("c 10000")
###### dump statistics
if ruby:
run_sim_command('ruby0.dump-stats "%s.stats"' % filename_prefix)
if generate_cache_data_dump:
run_sim_command('ruby0.dump-cache-data 0 "%s.cache"' % filename_prefix)
if opal:
run_sim_command('opal0.listparam')
run_sim_command('opal0.stats')
run_sim_command('opal0.sim-stop')
if checkpoint_at_end == 'yes':
mf_write_configuration_cmd("/scratch/%s/%s-%dc-%dp-%dt-%d-caches" % (getpass.getuser(), checkpoint, chips, processors_per_chip, smt_threads, transactions))
# close trace file
if ruby and generate_request_trace == 'yes':
run_sim_command('ruby0.tracer-output-file ""')
###### display resource usage
print_resource_usage()
###### close xterm capture file
get_console().output_file = ""
###### finishing up
run_sim_command('quit')
else:
print "enter interactive mode..."
print "Normally, the following command lines will be issued:"
print ""
print " @from mfacet import *"
print " @setup_run_for_n_transactions(%d, %d) % (transactions, dump_interval)"
print ""
print " # (optionally)"
print " ruby0.load-caches <workload>.warmup.gz"
print " ruby0.tracer-output-file <filename>.trace.gz"
print ""
print " # (if ruby only)"
print " continue"
print " ruby0.dump-stats file.stats"
print ""
print " # (if ruby + opal)"
print " opal0.sim-step 100000000000"
print " opal0.listparam"
print " opal0.stats"
print " opal0.sim-stop"
print ""
print " quit"
workloads.set_var(env_dict, 'CHECKPOINT', "%s_warm-%dp-%dmb.check"%( workload_name, (chips*smt_threads*procs_per_chip), target_memory ) )
workloads.set_var(env_dict, 'PROTOCOL', protocol)
workloads.set_var(env_dict, 'PROCESSORS', chips* procs_per_chip* smt_threads)
workloads.set_var(env_dict, 'CHIPS', chips)
workloads.set_var(env_dict, 'PROCS_PER_CHIP', procs_per_chip)
workloads.set_var(env_dict, "SMT_THREADS", smt_threads)
workloads.set_var(env_dict, 'RESULTS_DIR', cvsroot_results)
workloads.set_var(env_dict, 'TRANSACTIONS', transactions)
workloads.set_var(env_dict, 'DUMP_INTERVAL', dump_interval)
workloads.set_var(env_dict, 'CONDORCLUSTER', '$(Cluster)')
workloads.set_var(env_dict, 'CONDORPROCESS', '$(Process)')
#workloads.set_var(env_dict, 'USE_LOCAL_MIRROR', 'yes')
condor_file.write("getenv = False\n")
workloads.check_requirements(env_dict)
condor_file.write("environment = %s\n" % workloads.get_condor_env_string(env_dict))
condor_file.write("\n")
## Files and directories
condor_file.write('initialdir = %s/home/%s/\n' % (cvsroot_simics, protocol))
filename_prefix = "%s/%s" % (cvsroot_results, workloads.get_output_file_name_prefix(env_dict, 1))
condor_file.write('filename = %s\n' % filename_prefix)
condor_file.write('error = $(filename).error\n')
condor_file.write('output = $(filename).output\n')
condor_file.write("\n")
## Run multiple copies of this datapoint with varying
## priority. This makes sure condor will try to run
def load_modules_and_run(fast_forward=0):
print "Python version: %s" % sys.version
###### Read simulation parameters
env_dict = workloads.prepare_env_dictionary(simics=1)
workloads.check_requirements(env_dict)
workloads.print_all_variables(env_dict)
protocol = workloads.get_var(env_dict, "PROTOCOL")
workload_name = workloads.get_var(env_dict, "WORKLOAD")
checkpoint_dir = workloads.get_var(env_dict, "CHECKPOINT_DIR")
checkpoint = workloads.get_var(env_dict, "CHECKPOINT")
chips = int(workloads.get_var(env_dict, "CHIPS"))
processors_per_chip = int(workloads.get_var(env_dict, "PROCS_PER_CHIP"))
smt_threads = int(workloads.get_var(env_dict, "SMT_THREADS"))
l2_banks = int(workloads.get_var(env_dict, "NUM_L2_BANKS"))
bandwidth = int(workloads.get_var(env_dict, "BANDWIDTH"))
results_dir = workloads.get_var(env_dict, "RESULTS_DIR")
transactions = int(workloads.get_var(env_dict, "TRANSACTIONS"))
dump_interval = int(workloads.get_var(env_dict, "DUMP_INTERVAL"))
condor_cluster = int(workloads.get_var(env_dict, "CONDORCLUSTER"))
condor_process = int(workloads.get_var(env_dict, "CONDORPROCESS"))
checkpoint_at_end = workloads.get_var(env_dict, "CHECKPOINT_AT_END")
generate_request_trace = workloads.get_var(env_dict, "GENERATE_TRACE")
generate_cache_data_dump = workloads.get_var(env_dict, "CACHE_DATA_DUMP")
protocol_option = workloads.get_var(env_dict, "PROTOCOL_OPTION")
network_topology = workloads.get_var(env_dict, "NETWORK_TOPOLOGY")
warmup_file = workloads.get_var(env_dict, "WARMUP_FILE")
opal_config_file = workloads.get_var(env_dict, "OPAL_CONFIG_FILE")
opal_config_name = workloads.get_var(env_dict, "OPAL_CONFIG_NAME")
interactive = workloads.get_var(env_dict, "INTERACTIVE")
random_seed = workloads.get_var(env_dict, "RANDOM_SEED")
use_local_mirror = workloads.get_var(env_dict, "USE_LOCAL_MIRROR")
if random_seed == None:
random_seed = (condor_cluster << 16) | condor_process
else:
random_seed = int(random_seed)
###### mandatory parameters
assert (protocol != None)
assert (workload_name != None)
assert (checkpoint != None)
assert (chips != None)
assert (processors_per_chip != None)
assert (smt_threads != None)
assert (results_dir != None)
assert (transactions != None)
assert (dump_interval != None)
assert (condor_cluster != None)
assert (condor_process != None)
ruby = None
if "ruby" in get_module_list():
ruby = 1 # true
opal = None
if "opal" in get_module_list():
opal = 1 # true
###### print out local host name to help with troubleshooting
print "Local host name:", string.strip(os.popen("hostname").readline())
###### init simics with a checkpoint
assert (checkpoint[0] == '/')
run_sim_command('read-configuration "%s/%s"' %
(checkpoint_dir, checkpoint))
if (fast_forward): run_sim_command("continue %d" % fast_forward)
###### Conserve memory by limiting how much memory each simics image object can use
#@mfacet.limit_all_images(256)
###### set simics parameters
# enable/disable STCs here
run_sim_command("istc-disable")
run_sim_command("dstc-disable")
run_sim_command("stc-status")
# disable breakpoint before we sync all processors
run_sim_command('magic-break-disable')
# always use 1 for better parallelism
old_switch_time = conf.sim.cpu_switch_time
print "Old cpu_switch_time: %d" % conf.sim.cpu_switch_time
conf.sim.cpu_switch_time = 1
# 8/28/2003 CM: stepping or continuing simics before loading opal results
# in a major performance loss. See me for more details.
if not opal:
run_sim_command(
"c %d" %
(10 * old_switch_time)) # sync CPUs after changing switch time
print "New cpu_switch_time: %d" % conf.sim.cpu_switch_time
# enable breakpoint
run_sim_command('magic-break-enable')
# enable IFETCH, i-fetch line size is controlled in checkpoint files
run_sim_command("instruction-fetch-mode instruction-fetch-trace")
run_sim_command("instruction-fetch-mode")
filename_prefix = "%s/%s" % (
results_dir, workloads.get_output_file_name_prefix(env_dict, 1))
prep_dir(filename_prefix)
###### capture xterm output in file
import mod_xterm_console_commands
mod_xterm_console_commands.cap_start_cmd(get_console(),
filename_prefix + ".xterm")
###### Load modules and set parameters
if opal:
run_sim_command('load-module opal')
if ruby:
run_sim_command("load-module ruby")
if network_topology:
run_sim_command('ruby0.setparam_str g_NETWORK_TOPOLOGY "%s"' %
network_topology)
if protocol_option:
run_sim_command('ruby0.setparam_str g_MASK_PREDICTOR_CONFIG "%s"' %
protocol_option)
run_sim_command("ruby0.setparam g_PROCS_PER_CHIP %d" %
processors_per_chip)
run_sim_command("ruby0.setparam g_NUM_L2_BANKS %d" % l2_banks)
run_sim_command("ruby0.setparam g_RANDOM_SEED %d" % random_seed)
run_sim_command("ruby0.setparam g_endpoint_bandwidth %d" % bandwidth)
run_sim_command("ruby0.setparam g_NUM_PROCESSORS %d" %
(chips * processors_per_chip))
run_sim_command("ruby0.setparam g_NUM_SMT_THREADS %d" % smt_threads)
transactional_memory = 0
if transactional_memory:
run_sim_command(
"ruby0.setparam_str REMOVE_SINGLE_CYCLE_DCACHE_FAST_PATH true")
run_sim_command("ruby0.setparam_str PROFILE_EXCEPTIONS true")
run_sim_command("ruby0.setparam_str PROFILE_XACT true")
run_sim_command("ruby0.setparam_str XACT_MEMORY true")
run_sim_command("ruby0.setparam XACT_MAX_DEPTH %s" %
xact_max_depth)
#run_sim_command("ruby0.setparam_str DATA_BLOCK true")
run_sim_command("ruby0.setparam RETRY_LATENCY 10")
#run_sim_command("ruby0.setparam RANDOM_SEED 0")
run_sim_command("ruby0.setparam g_MEMORY_SIZE_BYTES 8589934592")
run_sim_command("ruby0.setparam g_DEADLOCK_THRESHOLD 20000000")
run_sim_command("ruby0.init")
run_sim_command("ruby0.periodic-stats-interval %d" % 1000000)
run_sim_command('ruby0.periodic-stats-file "%s.periodic" ' %
filename_prefix)
debug = 0
if debug:
run_sim_command('ruby0.debug-verb high')
run_sim_command('ruby0.debug-filter lseagTSN')
#run_sim_command('ruby0.debug-start-time "500000"')
run_sim_command('ruby0.debug-start-time "0"')
###### init opal
if opal:
# For debugging with symtable, used by Min's race detector
#run_sim_command('load-module symtable')
#run_sim_command('new-symtable mysql')
# set up the number of SMT Threads per physical processor::
run_sim_command("opal0.setparam CONFIG_LOGICAL_PER_PHY_PROC %d" %
smt_threads)
# set up the number of physical registers needed, for each physical register type:
run_sim_command("opal0.setparam CONFIG_IREG_PHYSICAL %d" %
(160 * smt_threads + 64))
run_sim_command("opal0.setparam CONFIG_FPREG_PHYSICAL %d" %
(64 * smt_threads + 128))
run_sim_command("opal0.setparam CONFIG_CCREG_PHYSICAL %d" %
(5 * smt_threads + 64))
if opal_config_file:
run_sim_command('opal0.init "%s"' % opal_config_file)
else:
run_sim_command('opal0.init')
run_sim_command('opal0.sim-start "%s.opal"' % filename_prefix)
###### do the following only when it is not interactive simulation
if not interactive:
###### warm up the caches
if ruby and warmup_file:
print "Warming caches with %s" % warmup_file
run_sim_command('ruby0.load-caches "%s/%s"' %
(checkpoint_dir, warmup_file))
###### generate request trace
if ruby and generate_request_trace:
trace_filename = "/scratch/%s/traces/%s-%dc-%dp-%dt-%s-%d-%d-%d.trace.gz" % (
getpass.getuser(), workload_name, chips, processors_per_chip,
smt_threads, protocol, bandwidth, condor_cluster,
condor_process)
prep_dir(trace_filename)
run_sim_command("ruby0.tracer-output-file %s" % trace_filename)
###### Run for n transactions or just a short run
print "Initial memory usage: %s" % memory_usage_str()
print "Running..."
if transactions > 0:
assert (dump_interval > 0)
setup_run_for_n_transactions(transactions, dump_interval)
if opal:
run_sim_command("opal0.sim-step 100000000000")
else:
run_sim_command("c")
else:
if opal:
run_sim_command("opal0.sim-step 1000")
else:
run_sim_command("c 10000")
###### dump statistics
if ruby:
run_sim_command('ruby0.dump-stats "%s.stats"' % filename_prefix)
if generate_cache_data_dump:
run_sim_command('ruby0.dump-cache-data 0 "%s.cache"' %
filename_prefix)
if opal:
run_sim_command('opal0.listparam')
run_sim_command('opal0.stats')
run_sim_command('opal0.sim-stop')
if checkpoint_at_end == 'yes':
mf_write_configuration_cmd(
"/scratch/%s/%s-%dc-%dp-%dt-%d-caches" %
(getpass.getuser(), checkpoint, chips, processors_per_chip,
smt_threads, transactions))
# close trace file
if ruby and generate_request_trace == 'yes':
run_sim_command('ruby0.tracer-output-file ""')
###### display resource usage
print_resource_usage()
###### close xterm capture file
get_console().output_file = ""
###### finishing up
run_sim_command('quit')
else:
print "enter interactive mode..."
print "Normally, the following command lines will be issued:"
print ""
print " @from mfacet import *"
print " @setup_run_for_n_transactions(%d, %d) % (transactions, dump_interval)"
print ""
print " # (optionally)"
print " ruby0.load-caches <workload>.warmup.gz"
print " ruby0.tracer-output-file <filename>.trace.gz"
print ""
print " # (if ruby only)"
print " continue"
print " ruby0.dump-stats file.stats"
print ""
print " # (if ruby + opal)"
print " opal0.sim-step 100000000000"
print " opal0.listparam"
print " opal0.stats"
print " opal0.sim-stop"
print ""
print " quit"