def create_spec_2017_fs_run(params):
kernel = params['kernel']
cpu = params['cpu']
workload = params['workload']
size = params['size']
if cpu == "kvm":
timeout = 24 * 60 * 60 # 1 day
else:
timeout = 20 * 24 * 60 * 60 # 20 days
gem5run = gem5Run.createFSRun(
'spec-2017;' + RUN_NAME_SUFFIX, # name
get_gem5_binary_path('classic'), # gem5_binary
os.path.join(GEM5_RESOURCES_FOLDER,
'src/spec-2017/configs/run_spec.py'), # run_script
os.path.join(
OUTPUT_FOLDER,
'spec-2017/{}/{}/{}/{}/'.format(kernel, cpu, workload,
size)), # outdir
gem5_binaries['classic'], # gem5_artifact
gem5_repo, # gem5_git_artifact
experiments_repo, # run_script_git_artifact
os.path.join(LINUX_KERNELS_FOLDER,
'vmlinux' + '-' + kernel), # linux_binary
os.path.join(DISK_IMAGES_FOLDER, 'spec-2017'), # disk_image
linux_binaries[kernel], # linux_binary_artifact
spec_2017_artifacts.disk_image, # disk_image_artifact
cpu,
workload,
size, # params
timeout=timeout,
check_failure=lambda run_obj: linux_booting_check_failure(run_obj))
return gem5run
def createRun(linux, boot_type, cpu, num_cpu, mem):
if mem in gem5_artifacts:
artifact_gem5 = gem5_artifacts[mem]
else:
# We can use any binary for classic
artifact_gem5 = gem5_artifacts[ruby_mem_types[0]]
binary_gem5 = artifact_gem5.path
return gem5Run.createFSRun(
'boot experiments with gem5-20',
binary_gem5,
'configs-boot-tests/run_exit.py',
'results/run_exit/vmlinux-{}/boot-exit/{}/{}/{}/{}'.format(
linux, cpu, mem, num_cpu, boot_type),
artifact_gem5,
gem5_repo,
experiments_repo,
os.path.join('linux-stable', 'vmlinux' + '-' + linux),
'disk-image/boot-exit/boot-exit-image/boot-exit',
linux_binaries[linux],
disk_image,
cpu,
mem,
num_cpu,
boot_type,
timeout=6 * 60 * 60 #6 hours
)
def createRun_realGraph(cpu, num_cpu, mem, workload, graph):
if mem == 'MESI_Two_Level':
binary_gem5 = 'gem5/build/X86_MESI_Two_Level/gem5.opt'
artifact_gem5 = gem5_binary_MESI_Two_Level
else:
binary_gem5 = 'gem5/build/X86/gem5.opt'
artifact_gem5 = gem5_binary
return gem5Run.createFSRun(
'Running GAPBS with gem5',
binary_gem5,
'configs-gapbs-tests/gapbs_config.py',
'results/run_exit/vmlinux-5.2.3/gapbs/{}/{}/{}/{}/real_graph/{}'.
format(cpu, num_cpu, mem, workload, size),
artifact_gem5,
gem5_repo,
experiments_repo,
'linux-stable/vmlinux-5.2.3',
'disk-image/gapbs-image/gapbs',
linux_binaries,
disk_image,
cpu,
num_cpu,
mem,
workload,
'0',
graph,
timeout=6 * 60 * 60)
def createRun(linux, boot_type, cpu, num_cpu, mem):
if mem == 'MESI_Two_Level':
binary_gem5 = 'gem5/build/X86_MESI_Two_Level/gem5.opt'
artifact_gem5 = gem5_binary_MESI_Two_Level
elif mem == 'MOESI_CMP_directory':
binary_gem5 = 'gem5/build/MOESI_CMP_directory/gem5.opt'
artifact_gem5 = gem5_binary_MOESI_CMP_directory
else:
binary_gem5 = 'gem5/build/X86/gem5.opt'
artifact_gem5 = gem5_binary
return gem5Run.createFSRun(
'boot experiments with gem5-20.1 (timeout reruns)',
binary_gem5,
'configs-boot-tests/run_exit.py',
'results/run_exit/vmlinux-{}/boot-exit/{}/{}/{}/{}'.format(
linux, cpu, mem, num_cpu, boot_type),
artifact_gem5,
gem5_repo,
experiments_repo,
os.path.join('linux-stable', 'vmlinux' + '-' + linux),
'disk-image/boot-exit/boot-exit-image/boot-exit',
linux_binaries[linux],
disk_image,
cpu,
mem,
num_cpu,
boot_type,
timeout=24 * 60 * 60 #10 hours
)
def createRun(bench, clas, cpu, mem, num_cpu):
if mem == 'MESI_Two_Level':
binary_gem5 = 'gem5/build/X86_MESI_Two_Level/gem5.opt'
artifact_gem5 = gem5_binary_MESI_Two_Level
else:
binary_gem5 = 'gem5/build/X86/gem5.opt'
artifact_gem5 = gem5_binary
return gem5Run.createFSRun(
'npb with gem5-20.1',
binary_gem5,
'configs-npb-tests-gem5-20.1/run_npb.py',
f'''results/run_npb_multicore/{bench}/{clas}/{cpu}/{num_cpu}''',
artifact_gem5,
gem5_repo,
experiments_repo,
'linux-stable/vmlinux-4.19.83',
'disk-image/npb/npb-image/npb',
linux_binary,
disk_image,
cpu,
mem,
bench.replace('.x', f'.{clas}.x'),
num_cpu,
timeout=240 * 60 * 60 #240 hours
)
def createRun(bench, size, cpu):
if cpu == 'timing' and size != 'simsmall':
return
return gem5Run.createFSRun(
'parsec classic memory tests with gem5-20',
'gem5/build/X86/gem5.opt',
'configs-parsec-tests/run_parsec.py',
f'''results/run_parsec/{bench}/{size}/{cpu}''',
gem5_binary, gem5_repo, experiments_repo,
'linux-stable/vmlinux-4.19.83',
'disk-image/parsec/parsec-image/parsec',
linux_binary, disk_image,
cpu, bench, size, '1',
timeout = 24*60*60 #24 hours
)
def createRun(bench, size, cpu, cores):
if cpu == 'timing' and (size != 'simsmall' or cores =='8'):
return
return gem5Run.createFSRun(
'parsec Ruby memory tests (MESI_Two_Level) with gem5-20',
'gem5/build/X86_MESI_Two_Level/gem5.opt',
'configs-parsec-tests-mesi-two-level/run_parsec_mesi_two_level.py',
f'''results/run_parsec_mesi_two_level/{bench}/{size}/{cpu}/{cores}''',
gem5_binary, gem5_repo, experiments_repo,
'linux-stable/vmlinux-4.19.83',
'disk-image/parsec/parsec-image/parsec',
linux_binary, disk_image,
cpu, bench, size, cores,
timeout = 24*60*60 #24 hours
)
def create_parsec_20_04_fs_run(params):
kernel = params['kernel']
cpu = params['cpu']
num_cpu = params['num_cpu']
mem_sys = params['mem_sys']
workload = params['workload']
size = params['size']
if cpu == "kvm":
timeout = 24 * 60 * 60 # 1 day
else:
timeout = 10 * 24 * 60 * 60 # 10 days
if mem_sys == "classic":
run_script = os.path.join(GEM5_RESOURCES_FOLDER,
"src/parsec/configs/run_parsec.py")
else:
run_script = os.path.join(
GEM5_RESOURCES_FOLDER,
"src/parsec/configs-mesi-two-level/run_parsec_mesi_two_level.py")
gem5run = gem5Run.createFSRun(
'parsec-20.04;' + RUN_NAME_SUFFIX, # name
get_gem5_binary_path(mem_sys), # gem5_binary
run_script, # run_script
os.path.join(
OUTPUT_FOLDER,
'parsec-20.04/{}/{}/{}/{}/{}/{}/'.format(kernel, cpu, num_cpu,
mem_sys, workload,
size)), # outdir
gem5_binaries[mem_sys], # gem5_artifact
gem5_repo, # gem5_git_artifact
experiments_repo, # run_script_git_artifact
os.path.join(LINUX_KERNELS_FOLDER,
'vmlinux' + '-' + kernel), # linux_binary
os.path.join(DISK_IMAGES_FOLDER, 'parsec-20.04'), # disk_image
linux_binaries[kernel], # linux_binary_artifact
parsec_20_04_artifacts.disk_image, # disk_image_artifact
cpu,
workload,
size,
num_cpu, # params
timeout=timeout,
check_failure=lambda run_obj: linux_booting_check_failure(run_obj))
return gem5run
def createRun(bench, size, cpu, cores):
return gem5Run.createFSRun(
'parsec_run_1_19_21_100INSTR',
'gem5/build/X86/gem5.opt',
'configs-parsec-tests/run_parsec.py',
f'''results/{cpu}/{cores}/{bench}/{size}''',
gem5_binary,
gem5_repo,
experiments_repo,
'linux-stable/vmlinux-4.19.83',
'disk-image/parsec/parsec-image/parsec',
linux_binary,
disk_image,
cpu,
bench,
size,
cores,
timeout=24 * 60 * 60 #3 hours
)
def create_boot_exit_fs_run(params):
kernel = params['kernel']
cpu = params['cpu']
mem_sys = params['mem_sys']
num_cpu = params['num_cpu']
boot_type = params['boot_type']
if cpu == "kvm":
timeout = 12 * 60 * 60 # 12 hours
else:
timeout = 2 * 24 * 60 * 60 # 2 days
assert (mem_sys in gem5_binaries)
gem5run = gem5Run.createFSRun(
'boot-exit;' + RUN_NAME_SUFFIX, # name
get_gem5_binary_path(mem_sys), # gem5_binary
os.path.join(GEM5_RESOURCES_FOLDER,
'src/boot-exit/configs/run_exit.py'), # run_script
os.path.join(
OUTPUT_FOLDER,
'boot-exit/{}/{}/{}/{}/{}/'.format(kernel, cpu, num_cpu, mem_sys,
boot_type)), # outdir
gem5_binaries[mem_sys], # gem5_artifact
gem5_repo, # gem5_git_artifact
experiments_repo, # run_script_git_artifact
os.path.join(LINUX_KERNELS_FOLDER,
'vmlinux' + '-' + kernel), # linux_binary
os.path.join(DISK_IMAGES_FOLDER, 'boot-exit.img'), # disk_image
linux_binaries[kernel], # linux_binary_artifact
boot_exit_artifacts.disk_image, # disk_image_artifact
cpu,
mem_sys,
num_cpu,
boot_type, # params
timeout=timeout,
check_failure=lambda run_obj: linux_booting_check_failure(run_obj))
output_folder = os.path.join(
OUTPUT_FOLDER,
'boot-exit/{}/{}/{}/{}/{}/'.format(kernel, cpu, num_cpu, mem_sys,
boot_type))
#assert(os.path.exists(os.path.join(output_folder, "../")))
#assert(not os.path.exists(os.path.join(output_folder, "simout")))
return gem5run
def create_gapbs_fs_run(params):
kernel = params['kernel']
cpu = params['cpu']
num_cpu = params['num_cpu']
mem_sys = params['mem_sys']
workload = params['workload']
synthetic = params['synthetic']
n_nodes = params['n_nodes']
if cpu == "kvm":
timeout = 24 * 60 * 60 # 1 day
else:
timeout = 10 * 24 * 60 * 60 # 10 days
gem5run = gem5Run.createFSRun(
'gapbs;' + RUN_NAME_SUFFIX, # name
get_gem5_binary_path(mem_sys), # gem5_binary
os.path.join(GEM5_RESOURCES_FOLDER,
'src/gapbs/configs/run_gapbs.py'), # run_script
os.path.join(
OUTPUT_FOLDER,
'gapbs/{}/{}/{}/{}/{}/{}/{}/'.format(kernel, cpu, num_cpu, mem_sys,
workload, synthetic,
n_nodes)), # outdir
gem5_binaries[mem_sys], # gem5_artifact
gem5_repo, # gem5_git_artifact
experiments_repo, # run_script_git_artifact
os.path.join(LINUX_KERNELS_FOLDER,
'vmlinux' + '-' + kernel), # linux_binary
os.path.join(DISK_IMAGES_FOLDER, 'gapbs.img'), # disk_image
linux_binaries[kernel], # linux_binary_artifact
gapbs_artifacts.disk_image, # disk_image_artifact
cpu,
num_cpu,
mem_sys,
workload,
synthetic,
n_nodes, # params
timeout=timeout,
check_failure=lambda run_obj: linux_booting_check_failure(run_obj))
return gem5run
'is.x', 'ep.x', 'cg.x', 'mg.x', 'ft.x', 'bt.x', 'sp.x', 'lu.x'
]
classes = ['A', 'B', 'C', 'D']
cpus = ['kvm', 'atomic']
for cpu in cpus:
for num_cpu in num_cpus:
for clas in classes:
for bm in benchmarks:
if cpu == 'atomic' and clas != 'A':
continue
run = gem5Run.createFSRun(
'npb_tests',
'gem5/build/X86/gem5.opt',
'configs-npb-tests/run_npb.py',
f'''results/run_npb/{bm}/{clas}/{cpu}/{num_cpu}''',
gem5_binary,
gem5_repo,
experiments_repo,
'linux-stable/vmlinux-4.19.83',
'disk-image/npb/npb-image/npb',
linux_binary,
disk_image,
cpu,
bm.replace('.x', f'.{clas}.x'),
num_cpu,
timeout=24 * 60 * 60 #24 hours
)
run_gem5_instance.apply_async((run, os.getcwd()))
num_cpus = ['1', '2', '4', '8']
cpu_types = ['kvm', 'atomic', 'simple', 'o3']
mem_types = ['classic', 'ruby']
for linux in linuxes:
for boot_type in boot_types:
for cpu in cpu_types:
for num_cpu in num_cpus:
for mem in mem_types:
run = gem5Run.createFSRun(
'linux_boot_tests',
'gem5/build/X86/gem5.opt',
'configs-boot-tests/run_exit.py',
'results/run_exit/vmlinux-{}/boot-exit/{}/{}/{}/{}'
.format(linux, cpu, mem, num_cpu, boot_type),
gem5_binary,
gem5_repo,
experiments_repo,
os.path.join('linux-stable',
'vmlinux' + '-' + linux),
'disk-image/boot-exit/boot-exit-image/boot-exit',
linux_binaries[linux],
disk_image,
cpu,
mem,
num_cpu,
boot_type,
timeout=12 * 60 * 60 #12 hours
)
run_gem5_instance.apply_async((run, os.getcwd()))
if mem == 'MESI_Two_Level':
binary_gem5 = 'gem5/build/X86_MESI_Two_Level/gem5.opt'
artifact_gem5 = gem5_binary_MESI_Two_Level
else:
binary_gem5 = 'gem5/build/X86/gem5.opt'
artifact_gem5 = gem5_binary
for size in benchmark_sizes[cpu]:
for benchmark in benchmarks:
run = gem5Run.createFSRun(
f'gem5 skylake config spec2006 runs gem5v20.1',
binary_gem5, # gem5_binary
'gem5-configs/run-spec-fs.py', # run_script
f'{result_dir}/{cpu}/{mem}/{size}/{benchmark}', # relative_outdir
artifact_gem5, # gem5_artifact
gem5_repo, # gem5_git_artifact
run_script_repo, # run_script_git_artifact
'linux-4.19.83/vmlinux-4.19.83', # linux_binary
'disk-image/spec-2006/spec-2006-image/spec-2006', # disk_image
linux_binary, # linux_binary_artifact
disk_image, # disk_image_artifact
cpu,
mem,
benchmark,
size, # params
timeout=days * 24 * 60 * 60 # in seconds
)
jobs.append(run)
with mp.Pool(mp.cpu_count() // 6) as pool:
pool.map(worker, jobs)
cpus = ['kvm', 'timing']
for cpu in cpus:
for num_cpu in num_cpus:
for size in sizes:
if cpu == 'timing' and size != 'simsmall':
continue
for bm in benchmarks:
run = gem5Run.createFSRun(
'parsec_tests',
'gem5/build/X86/gem5.opt',
'configs-parsec-tests/run_parsec.py',
f'''results/run_parsec/{bm}/{size}/{cpu}/{num_cpu}''',
gem5_binary,
gem5_repo,
experiments_repo,
'linux-stable/vmlinux-4.19.83',
'disk-image/parsec/parsec-image/parsec',
linux_binary,
disk_image,
cpu,
bm,
size,
num_cpu,
timeout=24 * 60 * 60 #24 hours
)
run_gem5_instance.apply_async((
run,
os.getcwd(),
))
'401.bzip2', '403.gcc', '410.bwaves', '416.gamess', '429.mcf',
'433.milc', '434.zeusmp', '435.gromacs', '436.cactusADM',
'437.leslie3d', '444.namd', '445.gobmk', '453.povray', '454.calculix',
'456.hmmer', '458.sjeng', '459.GemsFDTD', '462.libquantum',
'464.h264ref', '465.tonto', '470.lbm', '471.omnetpp', '473.astar',
'481.wrf', '482.sphinx3', '998.specrand', '999.specrand'
]
# unavailable benchmarks: 400.perlbench,447.dealII,450.soplex,483.xalancbmk
for cpu in cpus:
for size in benchmark_sizes[cpu]:
for benchmark in benchmarks:
run = gem5Run.createFSRun(
'gem5/build/X86/gem5.opt', # gem5_binary
'gem5-configs/run_spec.py', # run_script
'results/{}/{}/{}'.format(cpu, size,
benchmark), # relative_outdir
gem5_binary, # gem5_artifact
gem5_repo, # gem5_git_artifact
run_script_repo, # run_script_git_artifact
'linux-4.19.83/vmlinux-4.19.83', # linux_binary
'disk-image/spec2006/spec2006-image/spec2006', # disk_image
linux_binary, # linux_binary_artifact
disk_image, # disk_image_artifact
cpu,
benchmark,
size, # params
timeout=5 * 24 * 60 * 60 # 5 days
)
run_gem5_instance.apply_async((run, ))
if __name__ == "__main__":
boot_types = ['systemd']
num_cpus = ['1']
#cpu_types = ['kvm', 'atomic', 'simple', 'o3']
cpu_types = ['kvm']
mem_types = ['ruby']
for boot_type in boot_types:
for cpu in cpu_types:
for num_cpu in num_cpus:
for mem in mem_types:
run = gem5Run.createFSRun(
'gpu-fs-test',
'gem5/build/X86_MESI_Two_Level/gem5.opt',
'gem5-configs/run_fs.py',
'results/gpu-fs/',
gem5_binary,
gem5_repo,
experiments_repo,
'disk-image/gpu-fs/vmlinux',
'disk-image/gpu-fs/gpu-fs-image/gpu-fs',
linux_binary,
disk_image,
cpu,
mem,
num_cpu,
boot_type,
timeout=6 * 60 * 60 #6 hours
)
run_gem5_instance.apply_async((run, ))
