def install(
architecture: str,
channels: list,
versions: str,
verbose: bool,
install_dir: str = None) -> None:
'''
Downloads dotnet cli into the tools folder.
'''
__log_script_header("Downloading DotNet Cli")
if not install_dir:
install_dir = __get_directory(architecture)
if not path.exists(install_dir):
makedirs(install_dir)
getLogger().info("DotNet Install Path: '%s'", install_dir)
# Download appropriate dotnet install script
dotnetInstallScriptExtension = '.ps1' if platform == 'win32' else '.sh'
dotnetInstallScriptName = 'dotnet-install' + dotnetInstallScriptExtension
url = 'https://dot.net/v1/'
dotnetInstallScriptUrl = url + dotnetInstallScriptName
dotnetInstallScriptPath = path.join(install_dir, dotnetInstallScriptName)
getLogger().info('Downloading %s', dotnetInstallScriptUrl)
count = 0
while count < 3:
try:
with urlopen(dotnetInstallScriptUrl) as response:
if "html" in response.info()['Content-Type']:
count = count + 1
sleep(1) # sleep one second
continue
with open(dotnetInstallScriptPath, 'wb') as outfile:
outfile.write(response.read())
break
except Exception as ex:
count = count + 1
print(f"Attempt#{count} : {str(ex)}")
sleep(1)
continue
if count == 3:
getLogger().error("Fatal error: could not download dotnet-install script")
raise Exception("Fatal error: could not download dotnet-install script")
if platform != 'win32':
chmod(dotnetInstallScriptPath, S_IRWXU)
dotnetInstallInterpreter = [
'powershell.exe',
'-NoProfile',
'-ExecutionPolicy', 'Bypass',
dotnetInstallScriptPath
] if platform == 'win32' else [dotnetInstallScriptPath]
# If Version is supplied, pull down the specified version
common_cmdline_args = dotnetInstallInterpreter + [
'-InstallDir', install_dir,
'-Architecture', architecture
]
# Install Runtime/SDKs
if versions:
for version in versions:
cmdline_args = common_cmdline_args + ['-Version', version]
RunCommand(cmdline_args, verbose=verbose, retry=1).run(
get_repo_root_path()
)
# Only check channels if versions are not supplied.
# When we supply a version, but still pull down with -Channel, we will use
# whichever sdk is newer. So if we are trying to check an older version,
# or if there is a new version between when we start a run and when we actually
# run, we will be testing the "wrong" version, ie, not the version we specified.
if (not versions) and channels:
for channel in channels:
cmdline_args = common_cmdline_args + ['-Channel', channel]
RunCommand(cmdline_args, verbose=verbose, retry=1).run(
get_repo_root_path()
)
# Set DotNet Cli environment variables.
environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1'
environ['DOTNET_MULTILEVEL_LOOKUP'] = '0'
environ['UseSharedCompilation'] = 'false'
environ['DOTNET_ROOT'] = install_dir
# Add installed dotnet cli to PATH
environ["PATH"] = install_dir + pathsep + environ["PATH"]
# If we have copied dotnet from a different machine, then it may not be
# marked as executable. Fix this.
if platform != 'win32':
chmod(path.join(install_dir, 'dotnet'), S_IRWXU)
def install(architecture: str,
channels: list,
versions: str,
verbose: bool,
install_dir: str = None) -> None:
'''
Downloads dotnet cli into the tools folder.
'''
__log_script_header("Downloading DotNet Cli")
if not install_dir:
install_dir = __get_directory(architecture)
if not path.exists(install_dir):
makedirs(install_dir)
getLogger().info("DotNet Install Path: '%s'", install_dir)
# Download appropriate dotnet install script
dotnetInstallScriptExtension = '.ps1' if platform == 'win32' else '.sh'
dotnetInstallScriptName = 'dotnet-install' + dotnetInstallScriptExtension
url = 'https://dot.net/v1/'
dotnetInstallScriptUrl = url + dotnetInstallScriptName
dotnetInstallScriptPath = path.join(install_dir, dotnetInstallScriptName)
getLogger().info('Downloading %s', dotnetInstallScriptUrl)
count = 0
while count < 3:
try:
with urlopen(dotnetInstallScriptUrl,
context=ssl._create_unverified_context()) as response:
if "html" in response.info()['Content-Type']:
count = count + 1
sleep(1) # sleep one second
continue
with open(dotnetInstallScriptPath, 'wb') as outfile:
outfile.write(response.read())
break
except Exception:
count = count + 1
sleep(1)
continue
if count == 3:
getLogger().error(
"Fatal error: could not download dotnet-install script")
raise Exception(
"Fatal error: could not download dotnet-install script")
if platform != 'win32':
chmod(dotnetInstallScriptPath, S_IRWXU)
dotnetInstallInterpreter = [
'powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass',
dotnetInstallScriptPath
] if platform == 'win32' else [dotnetInstallScriptPath]
# If Version is supplied, pull down the specified version
common_cmdline_args = dotnetInstallInterpreter + [
'-InstallDir', install_dir, '-Architecture', architecture
]
# Install Runtime/SDKs
if versions:
for version in versions:
cmdline_args = common_cmdline_args + ['-Version', version]
RunCommand(cmdline_args, verbose=verbose,
retry=1).run(get_repo_root_path())
# Only check channels if versions are not supplied.
# When we supply a version, but still pull down with -Channel, we will use
# whichever sdk is newer. So if we are trying to check an older version,
# or if there is a new version between when we start a run and when we actually
# run, we will be testing the "wrong" version, ie, not the version we specified.
if (not versions) and channels:
for channel in channels:
cmdline_args = common_cmdline_args + [
'-Channel', ChannelMap.get_branch(channel)
]
if ChannelMap.get_quality_from_channel(channel) is not None:
cmdline_args += [
'-Quality',
ChannelMap.get_quality_from_channel(channel)
]
RunCommand(cmdline_args, verbose=verbose,
retry=1).run(get_repo_root_path())
setup_dotnet(install_dir)
def __main(args: list) -> int:
validate_supported_runtime()
args = __process_arguments(args)
verbose = not args.quiet
setup_loggers(verbose=verbose)
if not args.frameworks:
raise Exception("Framework version (-f) must be specified.")
target_framework_monikers = dotnet \
.FrameworkAction \
.get_target_framework_monikers(args.frameworks)
# Acquire necessary tools (dotnet)
init_tools(architecture=args.architecture,
dotnet_versions=args.dotnet_versions,
target_framework_monikers=target_framework_monikers,
verbose=verbose)
# WORKAROUND
# The MicroBenchmarks.csproj targets .NET Core 2.1, 3.0, 3.1 and 5.0
# to avoid a build failure when using older frameworks (error NETSDK1045:
# The current .NET SDK does not support targeting .NET Core $XYZ)
# we set the TFM to what the user has provided.
os.environ['PERFLAB_TARGET_FRAMEWORKS'] = ';'.join(
target_framework_monikers)
# dotnet --info
dotnet.info(verbose=verbose)
BENCHMARKS_CSPROJ = dotnet.CSharpProject(project=args.csprojfile,
bin_directory=args.bin_directory)
if not args.run_only:
# .NET micro-benchmarks
# Restore and build micro-benchmarks
micro_benchmarks.build(BENCHMARKS_CSPROJ, args.configuration,
target_framework_monikers, args.incremental,
verbose)
# Run micro-benchmarks
if not args.build_only:
upload_container = UPLOAD_CONTAINER
try:
for framework in args.frameworks:
micro_benchmarks.run(BENCHMARKS_CSPROJ, args.configuration,
framework, verbose, args)
globpath = os.path.join(
get_artifacts_directory() if not args.bdn_artifacts else
args.bdn_artifacts, '**', '*perf-lab-report.json')
except CalledProcessError:
upload_container = 'failedresults'
globpath = os.path.join(
get_artifacts_directory() if not args.bdn_artifacts else
args.bdn_artifacts, 'FailureReporter', 'failure-report.json')
cmdline = ['FailureReporting.exe', globpath]
reporterpath = os.path.join(helixpayload(), 'FailureReporter')
if not os.path.exists(reporterpath):
raise FileNotFoundError
RunCommand(cmdline, verbose=True).run(reporterpath)
dotnet.shutdown_server(verbose)
if args.upload_to_perflab_container:
import upload
upload.upload(globpath, upload_container, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def install(architecture: str,
channels: list,
versions: str,
verbose: bool,
install_dir: str = None) -> None:
'''
Downloads dotnet cli into the tools folder.
'''
__log_script_header("Downloading DotNet Cli")
if not install_dir:
install_dir = __get_directory(architecture)
if not path.exists(install_dir):
makedirs(install_dir)
getLogger().info("DotNet Install Path: '%s'", install_dir)
# Download appropriate dotnet install script
dotnetInstallScriptExtension = '.ps1' if platform == 'win32' else '.sh'
dotnetInstallScriptName = 'dotnet-install' + dotnetInstallScriptExtension
url = 'https://dot.net/v1/'
dotnetInstallScriptUrl = url + dotnetInstallScriptName
dotnetInstallScriptPath = path.join(install_dir, dotnetInstallScriptName)
getLogger().info('Downloading %s', dotnetInstallScriptUrl)
urlretrieve(dotnetInstallScriptUrl, dotnetInstallScriptPath)
if platform != 'win32':
chmod(dotnetInstallScriptPath, S_IRWXU)
dotnetInstallInterpreter = [
'powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass',
dotnetInstallScriptPath
] if platform == 'win32' else [dotnetInstallScriptPath]
# If Version is supplied, pull down the specified version
common_cmdline_args = dotnetInstallInterpreter + [
'-InstallDir', install_dir, '-Architecture', architecture
]
# Install Runtime/SDKs
if versions:
for version in versions:
cmdline_args = common_cmdline_args + ['-Version', version]
RunCommand(cmdline_args, verbose=verbose).run(get_repo_root_path())
# Only check channels if versions are not supplied.
# When we supply a version, but still pull down with -Channel, we will use
# whichever sdk is newer. So if we are trying to check an older version,
# or if there is a new version between when we start a run and when we actually
# run, we will be testing the "wrong" version, ie, not the version we specified.
if (not versions) and channels:
for channel in channels:
cmdline_args = common_cmdline_args + ['-Channel', channel]
RunCommand(cmdline_args, verbose=verbose).run(get_repo_root_path())
# Set DotNet Cli environment variables.
environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1'
environ['DOTNET_MULTILEVEL_LOOKUP'] = '0'
environ['UseSharedCompilation'] = 'false'
environ['DOTNET_ROOT'] = install_dir
# Add installed dotnet cli to PATH
environ["PATH"] = install_dir + pathsep + environ["PATH"]
# If we have copied dotnet from a different machine, it will not be executable. Fix this
if platform != 'win32':
chmod(path.join(install_dir, 'dotnet'), S_IRWXU)
def build(self,
configuration: str,
verbose: bool,
packages_path: str,
target_framework_monikers: list = None,
output_to_bindir: bool = False,
runtime_identifier: str = None,
*args) -> None:
'''Calls dotnet to build the specified project.'''
if not target_framework_monikers: # Build all supported frameworks.
cmdline = [
'dotnet',
'build',
self.csproj_file,
'--configuration',
configuration,
'--no-restore',
"/p:NuGetPackageRoot={}".format(packages_path),
'/p:UseSharedCompilation=false',
'/p:BuildInParallel=false',
'/m:1',
]
if output_to_bindir:
cmdline = cmdline + ['--output', self.__bin_directory]
if runtime_identifier:
cmdline = cmdline + ['--runtime', runtime_identifier]
if args:
cmdline = cmdline + list(args)
RunCommand(cmdline, verbose=verbose).run(self.working_directory)
else: # Only build specified frameworks
for target_framework_moniker in target_framework_monikers:
cmdline = [
'dotnet',
'build',
self.csproj_file,
'--configuration',
configuration,
'--framework',
target_framework_moniker,
'--no-restore',
"/p:NuGetPackageRoot={}".format(packages_path),
'/p:UseSharedCompilation=false',
'/p:BuildInParallel=false',
'/m:1',
]
if output_to_bindir:
cmdline = cmdline + ['--output', self.__bin_directory]
if runtime_identifier:
cmdline = cmdline + ['--runtime', runtime_identifier]
if args:
cmdline = cmdline + list(args)
RunCommand(cmdline,
verbose=verbose).run(self.working_directory)
def run(self):
'''
Runs the specified scenario
'''
self.parseargs()
if self.testtype == const.INNERLOOP:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.INNERLOOPMSBUILD:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.DOTNETWATCH:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH],
apptorun='dotnet',
appargs='watch -v',
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
self.traits.add_traits(workingdir=const.APPDIR)
startup.runtests(self.traits)
if self.testtype == const.STARTUP:
startup = StartupWrapper()
self.traits.add_traits(
overwrite=False,
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '',
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.STARTUP],
apptorun=publishedexe(self.traits.exename),
)
startup.runtests(self.traits)
elif self.testtype == const.SDK:
startup = StartupWrapper()
envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0'
envlistcleanbuild = ';'.join(
['MSBUILDDISABLENODEREUSE=1', envlistbuild])
# clean build
if self.sdktype == const.CLEAN_BUILD:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
apptorun=const.DOTNET,
appargs='build',
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR,
environmentvariables=envlistcleanbuild,
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# build(no changes)
if self.sdktype == const.BUILD_NO_CHANGE:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE),
apptorun=const.DOTNET,
appargs='build',
workingdir=const.APPDIR,
environmentvariables=envlistbuild)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# new console
if self.sdktype == const.NEW_CONSOLE:
self.traits.add_traits(
overwrite=False,
appargs='new console',
apptorun=const.DOTNET,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_new' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN:
startup = StartupWrapper()
crossgenexe = 'crossgen%s' % extension()
crossgenargs = self.crossgen_arguments.get_crossgen_command_line()
coreroot = self.crossgen_arguments.coreroot
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME,
workingdir=coreroot,
appargs=' '.join(crossgenargs))
self.traits.add_traits(
overwrite=False,
scenarioname='Crossgen Throughput - %s' % scenario_filename,
scenariotypename='%s - %s' %
(const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename),
apptorun='%s\%s' % (coreroot, crossgenexe),
)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN2:
startup = StartupWrapper()
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
crossgen2args = self.crossgen_arguments.get_crossgen2_command_line(
)
compiletype = self.crossgen_arguments.crossgen2_compiletype()
scenarioname = 'Crossgen2 Throughput - %s - %s' % (
compiletype, scenario_filename)
if self.crossgen_arguments.singlethreaded:
scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (
compiletype, scenario_filename)
if compiletype == const.CROSSGEN2_COMPOSITE:
self.traits.add_traits(overwrite=True, skipprofile='true')
self.traits.add_traits(
overwrite=True,
startupmetric=const.STARTUP_CROSSGEN2,
workingdir=self.crossgen_arguments.coreroot,
appargs='%s %s' % (os.path.join(
'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args)))
self.traits.add_traits(
overwrite=False,
scenarioname=scenarioname,
apptorun=os.path.join(self.crossgen_arguments.coreroot,
'corerun%s' % extension()),
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '' # turn on clr user events
)
startup.runtests(self.traits)
elif self.testtype == const.DEVICESTARTUP:
# ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc
# Regex used to split the response from starting the activity and saving each value
#Example:
# Starting: Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }
# Status: ok
# LaunchState: COLD
# Activity: net.dot.HelloAndroid/net.dot.MainActivity
# TotalTime: 241
# WaitTime: 242
# Complete
# Saves: [Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }, ok, COLD, net.dot.HelloAndroid/net.dot.MainActivity, 241, 242]
# Split results (start at 0) (List is Starting (Intent activity), Status (ok...), LaunchState ([HOT, COLD, WARM]), Activity (started activity name), TotalTime(toFrameOne), WaitTime(toFullLoad))
runSplitRegex = ":\s(.+)"
screenWasOff = False
getLogger().info("Clearing potential previous run nettraces")
for file in glob.glob(
os.path.join(const.TRACEDIR, 'PerfTest',
'runoutput.trace')):
if exists(file):
getLogger().info("Removed: " +
os.path.join(const.TRACEDIR, file))
os.remove(file)
cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb']
adb = RunCommand(cmdline, verbose=True)
adb.run()
# Do not remove, XHarness install seems to fail without an adb command called before the xharness command
getLogger().info("Preparing ADB")
cmdline = [adb.stdout.strip(), 'shell', 'wm', 'size']
RunCommand(cmdline, verbose=True).run()
# Get animation values
getLogger().info("Getting animation values")
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'window_animation_scale'
]
window_animation_scale_cmd = RunCommand(cmdline, verbose=True)
window_animation_scale_cmd.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'transition_animation_scale'
]
transition_animation_scale_cmd = RunCommand(cmdline, verbose=True)
transition_animation_scale_cmd.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'animator_duration_scale'
]
animator_duration_scale_cmd = RunCommand(cmdline, verbose=True)
animator_duration_scale_cmd.run()
getLogger().info(
f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}"
)
# Make sure animations are set to 1 or disabled
getLogger().info("Setting animation values")
if (self.animationsdisabled):
animationValue = 0
else:
animationValue = 1
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'window_animation_scale',
str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'transition_animation_scale',
str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'animator_duration_scale',
str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
# Check for success
getLogger().info("Getting animation values to verify it worked")
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'window_animation_scale'
]
windowSetValue = RunCommand(cmdline, verbose=True)
windowSetValue.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'transition_animation_scale'
]
transitionSetValue = RunCommand(cmdline, verbose=True)
transitionSetValue.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'animator_duration_scale'
]
animatorSetValue = RunCommand(cmdline, verbose=True)
animatorSetValue.run()
if (int(windowSetValue.stdout.strip()) != animationValue
or int(transitionSetValue.stdout.strip()) != animationValue
or int(animatorSetValue.stdout.strip()) != animationValue):
# Setting the values didn't work, error out
getLogger().exception(
f"Failed to set animation values to {animationValue}.")
exit(-1)
else:
getLogger().info(
f"Animation values successfully set to {animationValue}.")
installCmd = xharnesscommand() + [
self.devicetype, 'install', '--app', self.packagepath,
'--package-name', self.packagename, '-o', const.TRACEDIR, '-v'
]
RunCommand(installCmd, verbose=True).run()
getLogger().info("Completed install, running shell.")
cmdline = [
adb.stdout.strip(), 'shell',
f'cmd package resolve-activity --brief {self.packagename} | tail -n 1'
]
getActivity = RunCommand(cmdline, verbose=True)
getActivity.run()
getLogger().info(f"Target Activity {getActivity.stdout}")
# More setup stuff
checkScreenOnCmd = [
adb.stdout.strip(), 'shell',
f'dumpsys input_method | grep mInteractive'
]
checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True)
checkScreenOn.run()
keyInputCmd = [adb.stdout.strip(), 'shell', 'input', 'keyevent']
if ("mInteractive=false" in checkScreenOn.stdout):
# Turn on the screen to make interactive and see if it worked
getLogger().info("Screen was off, turning on.")
screenWasOff = True
RunCommand(keyInputCmd + ['26'],
verbose=True).run() # Press the power key
RunCommand(keyInputCmd + ['82'], verbose=True).run(
) # Unlock the screen with menu key (only works if it is not a password lock)
checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True)
checkScreenOn.run()
if ("mInteractive=false" in checkScreenOn.stdout):
getLogger().exception("Failed to make screen interactive.")
exit(-1)
# Actual testing some run stuff
getLogger().info("Test run to check if permissions are needed")
activityname = getActivity.stdout
startAppCmd = [
adb.stdout.strip(), 'shell', 'am', 'start-activity', '-W',
'-n', activityname
]
testRun = RunCommand(startAppCmd, verbose=True)
testRun.run()
testRunStats = re.findall(
runSplitRegex, testRun.stdout
) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime)
getLogger().info(f"Test run activity: {testRunStats[3]}")
stopAppCmd = [
adb.stdout.strip(), 'shell', 'am', 'force-stop',
self.packagename
]
RunCommand(stopAppCmd, verbose=True).run()
if "com.google.android.permissioncontroller" in testRunStats[3]:
# On perm screen, use the buttons to close it. it will stay away until the app is reinstalled
RunCommand(keyInputCmd + ['22'],
verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['22'],
verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['66'], verbose=True).run(
) # Press enter to close main perm screen
time.sleep(1)
RunCommand(keyInputCmd + ['22'],
verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['66'], verbose=True).run(
) # Press enter to close out of second screen
time.sleep(1)
# Check to make sure it worked
testRun = RunCommand(startAppCmd, verbose=True)
testRun.run()
testRunStats = re.findall(runSplitRegex, testRun.stdout)
getLogger().info(f"Test run activity: {testRunStats[3]}")
RunCommand(stopAppCmd, verbose=True).run()
if "com.google.android.permissioncontroller" in testRunStats[
3]:
getLogger().exception(
"Failed to get past permission screen, run locally to see if enough next button presses were used."
)
exit(-1)
allResults = []
for i in range(self.startupiterations):
startStats = RunCommand(startAppCmd, verbose=True)
startStats.run()
RunCommand(stopAppCmd, verbose=True).run()
allResults.append(
startStats.stdout
) # Save results (List is Intent, Status, LaunchState Activity, TotalTime, WaitTime)
time.sleep(3) # Delay in seconds for ensuring a cold start
getLogger().info("Stopping App for uninstall")
RunCommand(stopAppCmd, verbose=True).run()
getLogger().info("Uninstalling app")
uninstallAppCmd = xharnesscommand() + [
'android', 'uninstall', '--package-name', self.packagename
]
RunCommand(uninstallAppCmd, verbose=True).run()
# Reset animation values
getLogger().info("Resetting animation values to pretest values")
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'window_animation_scale',
window_animation_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'transition_animation_scale',
transition_animation_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'animator_duration_scale',
animator_duration_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
if screenWasOff:
RunCommand(keyInputCmd + ['26'],
verbose=True).run() # Turn the screen back off
# Create traces to store the data so we can keep the current general parse trace flow
getLogger().info(f"Logs: \n{allResults}")
os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True)
traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w")
for result in allResults:
traceFile.write(result)
traceFile.close()
startup = StartupWrapper()
self.traits.add_traits(
overwrite=True,
apptorun="app",
startupmetric=const.STARTUP_DEVICETIMETOMAIN,
tracefolder='PerfTest/',
tracename='runoutput.trace',
scenarioname=self.scenarioname)
startup.parsetraces(self.traits)
elif self.testtype == const.SOD:
sod = SODWrapper()
builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None
if not builtdir:
builtdir = const.BINDIR if os.path.exists(
const.BINDIR) else None
if not (self.dirs or builtdir):
raise Exception(
"Dirs was not passed in and neither %s nor %s exist" %
(const.PUBDIR, const.BINDIR))
sod.runtests(scenarioname=self.scenarioname,
dirs=self.dirs or builtdir,
artifact=self.traits.artifact)
def runtests(self, traits: TestTraits):
'''
Runs tests through startup
'''
# make sure required arguments are present
for key in ['apptorun', 'startupmetric', 'guiapp']:
if not getattr(traits, key):
raise Exception('startup tests require %s' % key)
defaultiterations = '1' if runninginlab() and not uploadtokenpresent() else '5' # only run 1 iteration for PR-triggered build
# required arguments & optional arguments with default values
startup_args = [
self.startuppath,
'--app-exe', traits.apptorun,
'--metric-type', traits.startupmetric,
'--trace-name', '%s_startup' % (traits.scenarioname or '%s_%s' % (traits.exename,traits.scenariotypename)),
'--gui-app', traits.guiapp,
'--process-will-exit', (traits.processwillexit or 'true'),
'--iterations', '%s' % (traits.iterations or defaultiterations),
'--timeout', '%s' % (traits.timeout or '50'),
'--warmup', '%s' % (traits.warmup or 'true'),
'--working-dir', '%s' % (traits.workingdir or sys.path[0]),
'--report-json-path', self.reportjson,
'--trace-directory', TRACEDIR
]
# optional arguments without default values
if traits.scenarioname:
startup_args.extend(['--scenario-name', traits.scenarioname])
if traits.appargs:
startup_args.extend(['--app-args', traits.appargs])
if traits.environmentvariables:
startup_args.extend(['--environment-variables', traits.environmentvariables])
if traits.iterationsetup:
startup_args.extend(['--iteration-setup', traits.iterationsetup])
if traits.setupargs:
startup_args.extend(['--setup-args', traits.setupargs])
if traits.iterationcleanup:
startup_args.extend(['--iteration-cleanup', traits.iterationcleanup])
if traits.cleanupargs:
startup_args.extend(['--cleanup-args', traits.cleanupargs])
if traits.measurementdelay:
startup_args.extend(['--measurement-delay', traits.measurementdelay])
if traits.skipprofile:
startup_args.extend(['--skip-profile-iteration'])
if traits.innerloopcommand:
startup_args.extend(['--inner-loop-command', traits.innerloopcommand])
if traits.innerloopcommandargs:
startup_args.extend(['--inner-loop-command-args', traits.innerloopcommandargs])
if traits.runwithoutexit:
startup_args.extend(['--run-without-exit', traits.runwithoutexit])
if traits.hotreloaditers:
startup_args.extend(['--hot-reload-iters', traits.hotreloaditers])
if traits.skipmeasurementiteration:
startup_args.extend(['--skip-measurement-iteration', traits.skipmeasurementiteration])
upload_container = UPLOAD_CONTAINER
try:
RunCommand(startup_args, verbose=True).run()
except CalledProcessError:
getLogger().info("Run failure registered")
# rethrow the original exception
raise
if runninginlab():
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
if uploadtokenpresent():
import upload
upload_code = upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
getLogger().info("Startup Upload Code: " + str(upload_code))
if upload_code != 0:
sys.exit(upload_code)
def shutdown_server(verbose: bool) -> None:
'''
Shuts down the dotnet server
'''
cmdline = ['dotnet', 'build-server', 'shutdown']
RunCommand(cmdline, verbose=verbose).run(get_repo_root_path())
def run(self):
'''
Runs the specified scenario
'''
self.parseargs()
if self.testtype == const.INNERLOOP:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.INNERLOOPMSBUILD:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.DOTNETWATCH:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH],
apptorun='dotnet',
appargs='watch -v',
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
self.traits.add_traits(workingdir=const.APPDIR)
startup.runtests(self.traits)
if self.testtype == const.STARTUP:
startup = StartupWrapper()
self.traits.add_traits(
overwrite=False,
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '',
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.STARTUP],
apptorun=publishedexe(self.traits.exename),
)
startup.runtests(self.traits)
elif self.testtype == const.SDK:
startup = StartupWrapper()
envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0'
envlistcleanbuild = ';'.join(
['MSBUILDDISABLENODEREUSE=1', envlistbuild])
# clean build
if self.sdktype == const.CLEAN_BUILD:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
apptorun=const.DOTNET,
appargs='build',
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR,
environmentvariables=envlistcleanbuild,
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# build(no changes)
if self.sdktype == const.BUILD_NO_CHANGE:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE),
apptorun=const.DOTNET,
appargs='build',
workingdir=const.APPDIR,
environmentvariables=envlistbuild)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# new console
if self.sdktype == const.NEW_CONSOLE:
self.traits.add_traits(
overwrite=False,
appargs='new console',
apptorun=const.DOTNET,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_new' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN:
startup = StartupWrapper()
crossgenexe = 'crossgen%s' % extension()
crossgenargs = self.crossgen_arguments.get_crossgen_command_line()
coreroot = self.crossgen_arguments.coreroot
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME,
workingdir=coreroot,
appargs=' '.join(crossgenargs))
self.traits.add_traits(
overwrite=False,
scenarioname='Crossgen Throughput - %s' % scenario_filename,
scenariotypename='%s - %s' %
(const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename),
apptorun='%s\%s' % (coreroot, crossgenexe),
)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN2:
startup = StartupWrapper()
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
crossgen2args = self.crossgen_arguments.get_crossgen2_command_line(
)
compiletype = self.crossgen_arguments.crossgen2_compiletype()
scenarioname = 'Crossgen2 Throughput - %s - %s' % (
compiletype, scenario_filename)
if self.crossgen_arguments.singlethreaded:
scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (
compiletype, scenario_filename)
if compiletype == const.CROSSGEN2_COMPOSITE:
self.traits.add_traits(overwrite=True, skipprofile='true')
self.traits.add_traits(
overwrite=True,
startupmetric=const.STARTUP_CROSSGEN2,
workingdir=self.crossgen_arguments.coreroot,
appargs='%s %s' % (os.path.join(
'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args)))
self.traits.add_traits(
overwrite=False,
scenarioname=scenarioname,
apptorun=os.path.join(self.crossgen_arguments.coreroot,
'corerun%s' % extension()),
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '' # turn on clr user events
)
startup.runtests(self.traits)
elif self.testtype == const.DEVICESTARTUP:
getLogger().info("Clearing potential previous run nettraces")
for file in glob.glob(
os.path.join(const.TRACEDIR, 'PerfTest',
'trace*.nettrace')):
if exists(file):
getLogger().info("Removed: " +
os.path.join(const.TRACEDIR, file))
os.remove(file)
cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb']
adb = RunCommand(cmdline, verbose=True)
adb.run()
cmdline = [
adb.stdout.strip(), 'shell', 'mkdir', '-p', '/sdcard/PerfTest'
]
RunCommand(cmdline, verbose=True).run()
cmdline = xharnesscommand() + [
self.devicetype, 'install', '--app', self.packagepath,
'--package-name', self.packagename, '-o', const.TRACEDIR, '-v'
]
RunCommand(cmdline, verbose=True).run()
for i in range(self.startupiterations):
cmdline = xharnesscommand() + [
'android', 'run', '-o', const.TRACEDIR, '--package-name',
self.packagename, '-v',
'--arg=env:COMPlus_EnableEventPipe=1',
'--arg=env:COMPlus_EventPipeOutputStreaming=1',
'--arg=env:COMPlus_EventPipeOutputPath=/sdcard/PerfTest/trace%s.nettrace'
% (i + 1), '--arg=env:COMPlus_EventPipeCircularMB=10',
'--arg=env:COMPlus_EventPipeConfig=Microsoft-Windows-DotNETRuntime:10:5',
'--expected-exit-code', self.expectedexitcode, '--dev-out',
'/sdcard/PerfTest/'
]
RunCommand(cmdline, verbose=True).run()
cmdline = xharnesscommand() + [
'android', 'uninstall', '--package-name', self.packagename
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'rm', '-r', '/sdcard/PerfTest'
]
RunCommand(cmdline, verbose=True).run()
startup = StartupWrapper()
self.traits.add_traits(
overwrite=True,
apptorun="app",
startupmetric=const.STARTUP_DEVICETIMETOMAIN,
tracefolder='PerfTest/',
tracename='trace*.nettrace',
scenarioname='Device Startup - Android %s' %
(self.packagename))
startup.parsetraces(self.traits)
elif self.testtype == const.SOD:
sod = SODWrapper()
builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None
if not builtdir:
builtdir = const.BINDIR if os.path.exists(
const.BINDIR) else None
if not (self.dirs or builtdir):
raise Exception(
"Dirs was not passed in and neither %s nor %s exist" %
(const.PUBDIR, const.BINDIR))
sod.runtests(scenarioname=self.scenarioname,
dirs=self.dirs or builtdir,
artifact=self.traits.artifact)
def upload_files(self, search_path: str):
self.download_azcopy()
cmdline = [
self.exe_path(), 'copy', search_path, self.get_upload_url(), '--recursive=true'
]
RunCommand(cmdline, verbose=self.verbose).run()
