def main(main_args):
"""Main entrypoint
Args:
main_args ([type]): Arguments to the script
"""
coreclr_args = setup_args(main_args)
antigen_directory = coreclr_args.antigen_directory
core_root = coreclr_args.core_root
tag_name = "{}-{}".format(coreclr_args.run_configuration,
coreclr_args.partition)
output_directory = coreclr_args.output_directory
run_duration = 120 # Run for 2 hours
path_to_corerun = os.path.join(core_root, "corerun")
path_to_tool = os.path.join(antigen_directory, "Antigen")
if is_windows:
path_to_corerun += ".exe"
path_to_tool += ".exe"
# Run tool such that issues are placed in a temp folder
with TempDir() as temp_location:
run_command([
path_to_tool, "-c", path_to_corerun, "-o", temp_location, "-d",
str(run_duration)
],
_exit_on_fail=True,
_long_running=True)
# Copy issues for upload
print("Copying issues to " + output_directory)
copy_issues(temp_location, output_directory, tag_name)
def build_and_run(coreclr_args):
"""Run perf scenarios under crank and collect data with SPMI"
Args:
coreclr_args (CoreClrArguments): Arguments use to drive
output_mch_name (string): Name of output mch file name
"""
source_directory = coreclr_args.source_directory
target_arch = coreclr_args.arch
target_os = coreclr_args.host_os
checked_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Checked")
release_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Release")
# We'll use repo script to install dotnet
dotnet_install_script_name = "dotnet-install.cmd" if is_windows else "dotnet-install.sh"
dotnet_install_script_path = path.join(source_directory, "eng", "common", dotnet_install_script_name)
with TempDir(skip_cleanup=True) as temp_location:
print ("Executing in " + temp_location)
# install dotnet 6.0
run_command([dotnet_install_script_path, "-Version", "6.0.4"], temp_location, _exit_on_fail=True)
os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0'
os.environ['DOTNET_SKIP_FIRST_TIME_EXPERIENCE'] = '1'
dotnet_path = path.join(source_directory, ".dotnet")
dotnet_exe = path.join(dotnet_path, "dotnet.exe") if is_windows else path.join(dotnet_path, "dotnet")
run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True)
os.environ['DOTNET_ROOT'] = dotnet_path
## install crank as local tool
run_command(
[dotnet_exe, "tool", "install", "Microsoft.Crank.Controller", "--version", "0.2.0-*", "--tool-path", temp_location], _exit_on_fail=True)
## ideally just do sparse clone, but this doesn't work locally
## git clone --filter=blob:none --no-checkout https://github.com/aspnet/benchmarks
## cd benchmarks
## git sparse-checkout init --cone
## git sparse-checkout set scenarios
## could probably just pass a URL and avoid this
run_command(
["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks"], temp_location, _exit_on_fail=True)
crank_app = path.join(temp_location, "crank")
mcs_path = determine_mcs_tool_path(coreclr_args)
superpmi_path = determine_superpmi_tool_path(coreclr_args)
# todo: add grpc/signalr, perhaps
configname_scenario_list = [
("platform", "plaintext"),
("json", "json"),
("plaintext", "mvc"),
("database", "fortunes_dapper"),
("database", "fortunes_ef_mvc_https"),
("database", "updates"),
("proxy", "proxy-yarp"),
("staticfiles", "static"),
("websocket", "websocket"),
("orchard", "about-sqlite")
]
# configname_scenario_list = [("platform", "plaintext")]
# note tricks to get one element tuples
runtime_options_list = [("Dummy=0",), ("TieredCompilation=0", ), ("TieredPGO=1",), ("TieredPGO=1", "ReadyToRun=0"),
("ReadyToRun=0", "OSR_HitLimit=0", "TC_OnStackReplacement_InitialCounter=10"),
("TieredPGO=1", "ReadyToRun=0", "OSR_HitLimit=0", "TC_OnStackReplacement_InitialCounter=10")]
# runtime_options_list = [("TieredCompilation=0", )]
mch_file = path.join(coreclr_args.output_mch_path, "aspnet.run." + target_os + "." + target_arch + ".checked.mch")
benchmark_machine = determine_benchmark_machine(coreclr_args)
jitname = determine_native_name(coreclr_args, "clrjit", target_os)
coreclrname = determine_native_name(coreclr_args, "coreclr", target_os)
spminame = determine_native_name(coreclr_args, "superpmi-shim-collector", target_os)
corelibname = "System.Private.CoreLib.dll"
jitpath = path.join(".", jitname)
jitlib = path.join(checked_root, jitname)
coreclr = path.join(release_root, coreclrname)
corelib = path.join(release_root, corelibname)
spmilib = path.join(checked_root, spminame)
for (configName, scenario) in configname_scenario_list:
configYml = configName + ".benchmarks.yml"
configFile = path.join(temp_location, "benchmarks", "scenarios", configYml)
crank_arguments = ["--config", configFile,
"--profile", benchmark_machine,
"--scenario", scenario,
"--application.framework", "net7.0",
"--application.channel", "edge",
"--application.sdkVersion", "latest",
"--application.environmentVariables", "COMPlus_JitName=" + spminame,
"--application.environmentVariables", "SuperPMIShimLogPath=.",
"--application.environmentVariables", "SuperPMIShimPath=" + jitpath,
"--application.environmentVariables", "COMPlus_EnableExtraSuperPmiQueries=1",
"--application.options.downloadFiles", "*.mc",
"--application.options.displayOutput", "true",
# "--application.options.dumpType", "full",
# "--application.options.fetch", "true",
"--application.options.outputFiles", spmilib,
"--application.options.outputFiles", jitlib,
"--application.options.outputFiles", coreclr,
"--application.options.outputFiles", corelib]
for runtime_options in runtime_options_list:
runtime_arguments = []
for runtime_option in runtime_options:
runtime_arguments.append("--application.environmentVariables")
runtime_arguments.append("COMPlus_" + runtime_option)
print("")
print("================================")
print("Config: " + configName + " scenario: " + scenario + " options: " + " ".join(runtime_options))
print("================================")
print("")
description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)]
subprocess.run([crank_app] + crank_arguments + description + runtime_arguments, cwd=temp_location)
# merge
command = [mcs_path, "-merge", "temp.mch", "*.mc", "-dedup", "-thin"]
run_command(command, temp_location)
# clean
command = [superpmi_path, "-v", "ewmi", "-f", "fail.mcl", jitlib, "temp.mch"]
run_command(command, temp_location)
# strip
if is_nonzero_length_file("fail.mcl"):
print("Replay had failures, cleaning...");
fail_file = path.join(coreclr_args.output_mch_path, "fail.mcl");
command = [mcs_path, "-strip", "fail.mcl", "temp.mch", mch_file]
run_command(command, temp_location)
else:
print("Replay was clean...");
shutil.copy2("temp.mch", mch_file)
# index
command = [mcs_path, "-toc", mch_file]
run_command(command, temp_location)
# overall summary
print("Merged summary for " + mch_file)
command = [mcs_path, "-jitflags", mch_file]
run_command(command, temp_location)
def main(main_args):
"""Main entrypoint
Args:
main_args ([type]): Arguments to the script
"""
coreclr_args = setup_args(main_args)
arch_name = coreclr_args.arch
os_name = "win" if coreclr_args.platform.lower() == "windows" else "linux"
run_configuration = "{}-{}".format(os_name, arch_name)
source_directory = coreclr_args.source_directory
# CorrelationPayload directories
correlation_payload_directory = path.join(coreclr_args.source_directory, "payload")
scripts_src_directory = path.join(source_directory, "src", "coreclr", 'scripts')
coreroot_dst_directory = path.join(correlation_payload_directory, "CoreRoot")
antigen_dst_directory = path.join(correlation_payload_directory, "exploratory")
helix_source_prefix = "official"
creator = ""
ci = True
if is_windows:
helix_queue = "Windows.10.Arm64" if arch_name == "arm64" else "Windows.10.Amd64.X86"
else:
if arch_name == "arm":
helix_queue = "(Ubuntu.1804.Arm32)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440"
elif arch_name == "arm64":
helix_queue = "(Ubuntu.1804.Arm64)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652"
else:
helix_queue = "Ubuntu.1804.Amd64"
# create exploratory directory
print('Copying {} -> {}'.format(scripts_src_directory, coreroot_dst_directory))
copy_directory(scripts_src_directory, coreroot_dst_directory, match_func=lambda path: any(path.endswith(extension) for extension in [".py"]))
if is_windows:
acceptable_copy = lambda path: any(path.endswith(extension) for extension in [".py", ".dll", ".exe", ".json"])
else:
# Need to accept files without any extension, which is how executable file's names look.
acceptable_copy = lambda path: (os.path.basename(path).find(".") == -1) or any(path.endswith(extension) for extension in [".py", ".dll", ".so", ".json", ".a"])
# copy CORE_ROOT
print('Copying {} -> {}'.format(coreclr_args.core_root_directory, coreroot_dst_directory))
copy_directory(coreclr_args.core_root_directory, coreroot_dst_directory, match_func=acceptable_copy)
try:
with TempDir() as tool_code_directory:
# clone the tool
run_command(
["git", "clone", "--quiet", "--depth", "1", "https://github.com/kunalspathak/Antigen.git", tool_code_directory])
antigen_bin_directory = path.join(tool_code_directory, "bin", "Release", "net5.0")
# build the tool
with ChangeDir(tool_code_directory):
dotnet_cmd = os.path.join(source_directory, "dotnet.cmd")
if not is_windows:
dotnet_cmd = os.path.join(source_directory, "dotnet.sh")
run_command([dotnet_cmd, "publish", "-c", "Release", "--self-contained", "-r", run_configuration, "-o", antigen_bin_directory], _exit_on_fail=True)
if not os.path.exists(path.join(antigen_bin_directory, "Antigen.dll")):
raise FileNotFoundError("Antigen.dll not present at {}".format(antigen_bin_directory))
# copy antigen tool
print('Copying {} -> {}'.format(antigen_bin_directory, antigen_dst_directory))
copy_directory(antigen_bin_directory, antigen_dst_directory, match_func=acceptable_copy)
except PermissionError as pe:
print("Skipping file. Got error: %s", pe)
# create foo.txt in work_item directories
workitem_directory = path.join(source_directory, "workitem")
os.mkdir(workitem_directory)
foo_txt = os.path.join(workitem_directory, "foo.txt")
with open(foo_txt, "w") as foo_txt_file:
foo_txt_file.write("hello world!")
# Set variables
print('Setting pipeline variables:')
set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory)
set_pipeline_variable("WorkItemDirectory", workitem_directory)
set_pipeline_variable("RunConfiguration", run_configuration)
set_pipeline_variable("Creator", creator)
set_pipeline_variable("Queue", helix_queue)
set_pipeline_variable("HelixSourcePrefix", helix_source_prefix)
def build_and_run(coreclr_args, output_mch_name):
"""Build the microbenchmarks and run them under "superpmi collect"
Args:
coreclr_args (CoreClrArguments): Arguments use to drive
output_mch_name (string): Name of output mch file name
"""
arch = coreclr_args.arch
python_path = sys.executable
core_root = coreclr_args.core_root
superpmi_directory = coreclr_args.superpmi_directory
performance_directory = coreclr_args.performance_directory
log_file = coreclr_args.log_file
partition_count = coreclr_args.partition_count
partition_index = coreclr_args.partition_index
dotnet_directory = os.path.join(performance_directory, "tools", "dotnet",
arch)
dotnet_exe = os.path.join(dotnet_directory, "dotnet")
artifacts_directory = os.path.join(performance_directory, "artifacts")
artifacts_packages_directory = os.path.join(artifacts_directory,
"packages")
project_file = path.join(performance_directory, "src", "benchmarks",
"micro", "MicroBenchmarks.csproj")
benchmarks_dll = path.join(artifacts_directory, "MicroBenchmarks.dll")
if is_windows:
shim_name = "%JitName%"
corerun_exe = "CoreRun.exe"
script_name = "run_microbenchmarks.bat"
else:
shim_name = "$JitName"
corerun_exe = "corerun"
script_name = "run_microbenchmarks.sh"
make_executable(dotnet_exe)
run_command([
dotnet_exe, "restore", project_file, "--packages",
artifacts_packages_directory
],
_exit_on_fail=True)
run_command([
dotnet_exe, "build", project_file, "--configuration", "Release",
"--framework", "net6.0", "--no-restore", "/p:NuGetPackageRoot=" +
artifacts_packages_directory, "-o", artifacts_directory
],
_exit_on_fail=True)
# Disable ReadyToRun so we always JIT R2R methods and collect them
collection_command = f"{dotnet_exe} {benchmarks_dll} --filter \"*\" --corerun {path.join(core_root, corerun_exe)} --partition-count {partition_count} " \
f"--partition-index {partition_index} --envVars COMPlus_JitName:{shim_name} " \
" COMPlus_ZapDisable:1 COMPlus_ReadyToRun:0 " \
"--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart"
# Generate the execution script in Temp location
with TempDir() as temp_location:
script_name = path.join(temp_location, script_name)
contents = []
# Unset the JitName so dotnet process will not fail
if is_windows:
contents.append("set JitName=%COMPlus_JitName%")
contents.append("set COMPlus_JitName=")
else:
contents.append("#!/bin/bash")
contents.append("export JitName=$COMPlus_JitName")
contents.append("unset COMPlus_JitName")
contents.append(f"pushd {performance_directory}")
contents.append(collection_command)
with open(script_name, "w") as collection_script:
collection_script.write(os.linesep.join(contents))
print()
print(f"{script_name} contents:")
print("******************************************")
print(os.linesep.join(contents))
print("******************************************")
make_executable(script_name)
run_command([
python_path,
path.join(superpmi_directory, "superpmi.py"), "collect",
"-core_root", core_root, "-output_mch_path", output_mch_name,
"-log_file", log_file, "-log_level", "debug", script_name
],
_exit_on_fail=True)
def copy_issues(issues_directory, upload_directory, tag_name):
"""Copies issue files (only top 5 smallest files from each folder) into the upload_directory
Args:
issues_directory (string): Issues directory
upload_directory (string): Upload directory
tag_name (string): Tag name for zip file
Returns:
[type]: [description]
"""
# Create upload_directory
if not os.path.isdir(upload_directory):
os.makedirs(upload_directory)
# Create temp directory to copy all issues to upload. We don't want to create a sub-folder
# inside issues_directory because that will also get included twice.
with TempDir() as prep_directory:
def sorter_by_size(pair):
""" Sorts the pair (file_name, file_size) tuple in ascending order of file_size
Args:
pair ([(string, int)]): List of tuple of file_name, file_size
"""
pair.sort(key=lambda x: x[1], reverse=False)
return pair
summary_of_summary = []
for file_path, dirs, files in walk(issues_directory, topdown=True):
filename_with_size = []
# Credit: https://stackoverflow.com/a/19859907
dirs[:] = [d for d in dirs]
for name in files:
if not name.lower().endswith(".g.cs"):
continue
curr_file_path = path.join(file_path, name)
size = getsize(curr_file_path)
filename_with_size.append((curr_file_path, size))
if len(filename_with_size) == 0:
continue
summary_file = path.join(file_path, "summary.txt")
summary_of_summary.append("**** " + file_path)
with open(summary_file, 'r') as sf:
summary_of_summary.append(sf.read())
filename_with_size.append(
(summary_file, 0)) # always copy summary.txt
# Copy atmost 5 files from each bucket
sorted_files = [
f[0] for f in sorter_by_size(filename_with_size)[:6]
] # sorter_by_size(filename_with_size)[:6]
print('### Copying below files from {0} to {1}:'.format(
issues_directory, prep_directory))
print('')
print(os.linesep.join(sorted_files))
for src_file in sorted_files:
dst_file = src_file.replace(issues_directory, prep_directory)
dst_directory = path.dirname(dst_file)
if not os.path.exists(dst_directory):
os.makedirs(dst_directory)
try:
shutil.copy2(src_file, dst_file)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))
issues_summary_file_name = "issues-summary-{}.txt".format(tag_name)
print("Creating {} in {}".format(issues_summary_file_name,
upload_directory))
issues_summary_file = os.path.join(upload_directory,
issues_summary_file_name)
with open(issues_summary_file, 'w') as sf:
sf.write(os.linesep.join(summary_of_summary))
# Also copy the issues-summary inside zip folder
dst_issue_summary_file = os.path.join(prep_directory,
issues_summary_file_name)
try:
shutil.copy2(issues_summary_file, dst_issue_summary_file)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))
# Zip compress the files we will upload
zip_path = os.path.join(prep_directory, "AllIssues-" + tag_name)
print("Creating archive: " + zip_path)
shutil.make_archive(zip_path, 'zip', prep_directory)
zip_path += ".zip"
dst_zip_path = os.path.join(upload_directory,
"AllIssues-" + tag_name + ".zip")
print("Copying {} to {}".format(zip_path, dst_zip_path))
try:
shutil.copy2(zip_path, dst_zip_path)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))