def __init__(self):
self.project: CSharpProject
self.projectfile: CSharpProjFile
self.crossgen_arguments = CrossgenArguments()
parser = ArgumentParser()
subparsers = parser.add_subparsers(
title='Operations',
description=
'Common preperation steps for perf tests. Should run under src\scenarios\<test asset folder>',
dest='operation')
default_parser = subparsers.add_parser(
DEFAULT,
help=
'Default operation (placeholder command and no specific operation will be executed)'
)
self.add_common_arguments(default_parser)
build_parser = subparsers.add_parser(BUILD, help='Builds the project')
self.add_common_arguments(build_parser)
publish_parser = subparsers.add_parser(PUBLISH,
help='Publishes the project')
self.add_common_arguments(publish_parser)
crossgen_parser = subparsers.add_parser(
CROSSGEN, help='Runs crossgen on a particular file')
self.add_common_arguments(crossgen_parser)
self.crossgen_arguments.add_crossgen_arguments(crossgen_parser)
crossgen2_parser = subparsers.add_parser(
CROSSGEN2, help='Runs crossgen2 on a particular file')
self.add_common_arguments(crossgen2_parser)
self.crossgen_arguments.add_crossgen2_arguments(crossgen2_parser)
args = parser.parse_args()
if not args.operation:
getLogger().error("Please specify an operation: %s" %
list(OPERATIONS))
sys.exit(1)
self.configuration = args.configuration
self.operation = args.operation
self.framework = args.framework
self.runtime_identifier = args.runtime
self.msbuild = args.msbuild
print(self.msbuild)
self.msbuildstatic = args.msbuildstatic
self.binlog = args.binlog
self.has_workload = args.has_workload
self.readonly_dotnet = args.readonly_dotnet
self.windows = args.windows
if self.operation == CROSSGEN:
self.crossgen_arguments.parse_crossgen_args(args)
if self.operation == CROSSGEN2:
self.crossgen_arguments.parse_crossgen2_args(args)
def __init__(self, traits: TestTraits):
self.traits = traits
self.testtype = None
self.sdktype = None
self.scenarioname = None
self.coreroot = None
self.crossgenfile = None
self.dirs = None
self.crossgen_arguments = CrossgenArguments()
setup_loggers(True)
class Runner:
'''
Wrapper for running all the things
'''
def __init__(self, traits: TestTraits):
self.traits = traits
self.testtype = None
self.sdktype = None
self.scenarioname = None
self.coreroot = None
self.crossgenfile = None
self.dirs = None
self.crossgen_arguments = CrossgenArguments()
setup_loggers(True)
def parseargs(self):
'''
Parses input args to the script
'''
parser = ArgumentParser(
description=
'test.py runs the test with specified commands. Usage: test.py <command> <optional subcommands> <options>',
formatter_class=RawTextHelpFormatter)
subparsers = parser.add_subparsers(
title='subcommands for scenario tests', dest='testtype')
# startup command
startupparser = subparsers.add_parser(
const.STARTUP,
description='measure time to main of running the project')
self.add_common_arguments(startupparser)
# sdk command
sdkparser = subparsers.add_parser(
const.SDK,
description='subcommands for sdk scenario',
formatter_class=RawTextHelpFormatter)
sdkparser.add_argument('sdktype',
choices=[
const.CLEAN_BUILD, const.BUILD_NO_CHANGE,
const.NEW_CONSOLE
],
type=str.lower,
help='''
clean_build: measure duration of building from source in each iteration
build_no_change: measure duration of building with existing output in each iteration
new_console: measure duration of creating a new console template
''')
self.add_common_arguments(sdkparser)
crossgenparser = subparsers.add_parser(
const.CROSSGEN,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen_arguments(crossgenparser)
self.add_common_arguments(crossgenparser)
crossgen2parser = subparsers.add_parser(
const.CROSSGEN2,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen2_arguments(crossgen2parser)
self.add_common_arguments(crossgen2parser)
sodparser = subparsers.add_parser(
const.SOD,
description=
'measure size on disk of the specified directory and its children')
sodparser.add_argument('--dirs',
dest='dirs',
type=str,
help=r'''
directories to measure separated by semicolon
ex: C:\repos\performance;C:\repos\runtime
''')
self.add_common_arguments(sodparser)
args = parser.parse_args()
if not args.testtype:
getLogger().error(
"Please specify a test type: %s. Type test.py <test type> -- help for more type-specific subcommands"
% testtypes)
sys.exit(1)
self.testtype = args.testtype
if self.testtype == const.SDK:
self.sdktype = args.sdktype
if self.testtype == const.CROSSGEN:
self.crossgen_arguments.parse_crossgen_args(args)
if self.testtype == const.CROSSGEN2:
self.crossgen_arguments.parse_crossgen2_args(args)
if self.testtype == const.SOD:
self.dirs = args.dirs
if args.scenarioname:
self.scenarioname = args.scenarioname
def add_common_arguments(self, parser: ArgumentParser):
"Common arguments to add to subparsers"
parser.add_argument('--scenario-name', dest='scenarioname')
def run(self):
'''
Runs the specified scenario
'''
self.parseargs()
if self.testtype == const.STARTUP:
startup = StartupWrapper()
self.traits.add_traits(
overwrite=False,
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '',
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.STARTUP],
apptorun=publishedexe(self.traits.exename),
)
startup.runtests(self.traits)
elif self.testtype == const.SDK:
startup = StartupWrapper()
envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0'
envlistcleanbuild = ';'.join(
['MSBUILDDISABLENODEREUSE=1', envlistbuild])
# clean build
if self.sdktype == const.CLEAN_BUILD:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
apptorun=const.DOTNET,
appargs='build',
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR,
environmentvariables=envlistcleanbuild,
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# build(no changes)
if self.sdktype == const.BUILD_NO_CHANGE:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE),
apptorun=const.DOTNET,
appargs='build',
workingdir=const.APPDIR,
environmentvariables=envlistbuild)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# new console
if self.sdktype == const.NEW_CONSOLE:
self.traits.add_traits(
overwrite=False,
appargs='new console',
apptorun=const.DOTNET,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_new' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN:
startup = StartupWrapper()
crossgenexe = 'crossgen%s' % extension()
crossgenargs = self.crossgen_arguments.get_crossgen_command_line()
coreroot = self.crossgen_arguments.coreroot
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME,
workingdir=coreroot,
appargs=' '.join(crossgenargs))
self.traits.add_traits(
overwrite=False,
scenarioname='Crossgen Throughput - %s' % scenario_filename,
scenariotypename='%s - %s' %
(const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename),
apptorun='%s\%s' % (coreroot, crossgenexe),
)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN2:
startup = StartupWrapper()
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
crossgen2args = self.crossgen_arguments.get_crossgen2_command_line(
)
compiletype = self.crossgen_arguments.crossgen2_compiletype()
scenarioname = 'Crossgen2 Throughput - %s - %s' % (
compiletype, scenario_filename)
if self.crossgen_arguments.singlethreaded:
scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (
compiletype, scenario_filename)
if compiletype == const.CROSSGEN2_COMPOSITE:
self.traits.add_traits(overwrite=True, skipprofile='true')
self.traits.add_traits(
overwrite=True,
startupmetric=const.STARTUP_CROSSGEN2,
workingdir=self.crossgen_arguments.coreroot,
appargs='%s %s' % (os.path.join(
'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args)))
self.traits.add_traits(
overwrite=False,
scenarioname=scenarioname,
apptorun=os.path.join(self.crossgen_arguments.coreroot,
'corerun%s' % extension()),
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '' # turn on clr user events
)
startup.runtests(self.traits)
elif self.testtype == const.SOD:
sod = SODWrapper()
builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None
if not builtdir:
builtdir = const.BINDIR if os.path.exists(
const.BINDIR) else None
if not (self.dirs or builtdir):
raise Exception(
"Dirs was not passed in and neither %s nor %s exist" %
(const.PUBDIR, const.BINDIR))
sod.runtests(scenarioname=self.scenarioname,
dirs=self.dirs or builtdir,
artifact=self.traits.artifact)
class Runner:
'''
Wrapper for running all the things
'''
def __init__(self, traits: TestTraits):
self.traits = traits
self.testtype = None
self.sdktype = None
self.scenarioname = None
self.coreroot = None
self.crossgenfile = None
self.dirs = None
self.crossgen_arguments = CrossgenArguments()
setup_loggers(True)
def parseargs(self):
'''
Parses input args to the script
'''
parser = ArgumentParser(description='test.py runs the test with specified commands. Usage: test.py <command> <optional subcommands> <options>',
formatter_class=RawTextHelpFormatter)
subparsers = parser.add_subparsers(title='subcommands for scenario tests',
dest='testtype')
# startup command
startupparser = subparsers.add_parser(const.STARTUP,
description='measure time to main of running the project')
self.add_common_arguments(startupparser)
# parse only command
parseonlyparser = subparsers.add_parser(const.DEVICESTARTUP,
description='measure time to startup for Android/iOS apps')
parseonlyparser.add_argument('--device-type', choices=['android','ios'],type=str.lower,help='Device type for testing', dest='devicetype')
parseonlyparser.add_argument('--package-path', help='Location of test application', dest='packagepath')
parseonlyparser.add_argument('--package-name', help='Classname (Android) or Bundle ID (iOS) of application', dest='packagename')
parseonlyparser.add_argument('--startup-iterations', help='Startups to run (1+)', type=int, default=10, dest='startupiterations')
parseonlyparser.add_argument('--disable-animations', help='Disable Android device animations, does nothing on iOS.', action='store_true', dest='animationsdisabled')
parseonlyparser.add_argument('--use-fully-drawn-time', help='Use the startup time from reportFullyDrawn for android, the equivalent for iOS is handled via logging a magic string and passing it to --fully-drawn-magic-string', action='store_true', dest='usefullydrawntime')
parseonlyparser.add_argument('--fully-drawn-extra-delay', help='Set an additional delay time for an Android app to reportFullyDrawn (seconds), not on iOS. This should be greater than the greatest amount of extra time expected between first frame draw and reportFullyDrawn being called. Default = 3 seconds', type=int, default=3, dest='fullyDrawnDelaySecMax')
parseonlyparser.add_argument('--fully-drawn-magic-string', help='Set the magic string that is logged by the app to indicate when the app is fully drawn. Required when using --use-fully-drawn-time on iOS.', type=str, dest='fullyDrawnMagicString')
self.add_common_arguments(parseonlyparser)
# inner loop command
innerloopparser = subparsers.add_parser(const.INNERLOOP,
description='measure time to main and difference between two runs in a row')
self.add_common_arguments(innerloopparser)
# inner loop msbuild command
innerloopparser = subparsers.add_parser(const.INNERLOOPMSBUILD,
description='measure time to main and difference between two runs in a row')
self.add_common_arguments(innerloopparser)
# dotnet watch command
dotnetwatchparser = subparsers.add_parser(const.DOTNETWATCH,
description='measure time to main and time for hot reload')
self.add_common_arguments(dotnetwatchparser)
# sdk command
sdkparser = subparsers.add_parser(const.SDK,
description='subcommands for sdk scenario',
formatter_class=RawTextHelpFormatter)
sdkparser.add_argument('sdktype',
choices=[const.CLEAN_BUILD, const.BUILD_NO_CHANGE, const.NEW_CONSOLE],
type=str.lower,
help=
'''
clean_build: measure duration of building from source in each iteration
build_no_change: measure duration of building with existing output in each iteration
new_console: measure duration of creating a new console template
'''
)
self.add_common_arguments(sdkparser)
crossgenparser = subparsers.add_parser(const.CROSSGEN,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen_arguments(crossgenparser)
self.add_common_arguments(crossgenparser)
crossgen2parser = subparsers.add_parser(const.CROSSGEN2,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen2_arguments(crossgen2parser)
self.add_common_arguments(crossgen2parser)
sodparser = subparsers.add_parser(const.SOD,
description='measure size on disk of the specified directory and its children')
sodparser.add_argument('--dirs',
dest='dirs',
type=str,
help=
r'''
directories to measure separated by semicolon
ex: C:\repos\performance;C:\repos\runtime
''' )
self.add_common_arguments(sodparser)
args = parser.parse_args()
if not args.testtype:
getLogger().error("Please specify a test type: %s. Type test.py <test type> -- help for more type-specific subcommands" % testtypes)
sys.exit(1)
self.testtype = args.testtype
if self.testtype == const.SDK:
self.sdktype = args.sdktype
if self.testtype == const.CROSSGEN:
self.crossgen_arguments.parse_crossgen_args(args)
if self.testtype == const.CROSSGEN2:
self.crossgen_arguments.parse_crossgen2_args(args)
if self.testtype == const.SOD:
self.dirs = args.dirs
if self.testtype == const.DEVICESTARTUP:
self.packagepath = args.packagepath
self.packagename = args.packagename
self.devicetype = args.devicetype
self.startupiterations = args.startupiterations
self.animationsdisabled = args.animationsdisabled
self.usefullydrawntime = args.usefullydrawntime
self.fullyDrawnDelaySecMax = args.fullyDrawnDelaySecMax
self.fullyDrawnMagicString = args.fullyDrawnMagicString
if args.scenarioname:
self.scenarioname = args.scenarioname
def add_common_arguments(self, parser: ArgumentParser):
"Common arguments to add to subparsers"
parser.add_argument('--scenario-name',
dest='scenarioname')
def run(self):
'''
Runs the specified scenario
'''
self.parseargs()
if self.testtype == const.INNERLOOP:
startup = StartupWrapper()
self.traits.add_traits(scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP],
apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.INNERLOOPMSBUILD:
startup = StartupWrapper()
self.traits.add_traits(scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD],
apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.DOTNETWATCH:
startup = StartupWrapper()
self.traits.add_traits(scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH],
apptorun='dotnet', appargs='watch -v',
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
self.traits.add_traits(workingdir = const.APPDIR)
startup.runtests(self.traits)
if self.testtype == const.STARTUP:
startup = StartupWrapper()
self.traits.add_traits(overwrite=False,
environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '',
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.STARTUP],
apptorun=publishedexe(self.traits.exename),
)
startup.runtests(self.traits)
elif self.testtype == const.SDK:
startup = StartupWrapper()
envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0'
envlistcleanbuild = ';'.join(['MSBUILDDISABLENODEREUSE=1', envlistbuild])
# clean build
if self.sdktype == const.CLEAN_BUILD:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
apptorun=const.DOTNET,
appargs='build',
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR,
environmentvariables=envlistcleanbuild,
)
self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# build(no changes)
if self.sdktype == const.BUILD_NO_CHANGE:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE),
apptorun=const.DOTNET,
appargs='build',
workingdir=const.APPDIR,
environmentvariables=envlistbuild
)
self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# new console
if self.sdktype == const.NEW_CONSOLE:
self.traits.add_traits(
overwrite=False,
appargs='new console',
apptorun=const.DOTNET,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR
)
self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN:
startup = StartupWrapper()
crossgenexe = 'crossgen%s' % extension()
crossgenargs = self.crossgen_arguments.get_crossgen_command_line()
coreroot = self.crossgen_arguments.coreroot
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename()
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME,
workingdir=coreroot,
appargs=' '.join(crossgenargs)
)
self.traits.add_traits(overwrite=False,
scenarioname='Crossgen Throughput - %s' % scenario_filename,
scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename),
apptorun='%s\%s' % (coreroot, crossgenexe),
)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN2:
startup = StartupWrapper()
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename()
crossgen2args = self.crossgen_arguments.get_crossgen2_command_line()
compiletype = self.crossgen_arguments.crossgen2_compiletype()
scenarioname = 'Crossgen2 Throughput - %s - %s' % (compiletype, scenario_filename)
if self.crossgen_arguments.singlethreaded:
scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (compiletype, scenario_filename)
if compiletype == const.CROSSGEN2_COMPOSITE:
self.traits.add_traits(overwrite=True,
skipprofile='true')
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_CROSSGEN2,
workingdir=self.crossgen_arguments.coreroot,
appargs='%s %s' % (os.path.join('crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args))
)
self.traits.add_traits(overwrite=False,
scenarioname=scenarioname,
apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()),
environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events
)
startup.runtests(self.traits)
elif self.testtype == const.DEVICESTARTUP and self.devicetype == 'android':
# ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc
# Regex used to split the response from starting the activity and saving each value
#Example:
# Starting: Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }
# Status: ok
# LaunchState: COLD
# Activity: net.dot.HelloAndroid/net.dot.MainActivity
# TotalTime: 241
# WaitTime: 242
# Complete
# Saves: [Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }, ok, COLD, net.dot.HelloAndroid/net.dot.MainActivity, 241, 242]
# Split results (start at 0) (List is Starting (Intent activity), Status (ok...), LaunchState ([HOT, COLD, WARM]), Activity (started activity name), TotalTime(toFrameOne), WaitTime(toFullLoad))
runSplitRegex = ":\s(.+)"
screenWasOff = False
getLogger().info("Clearing potential previous run nettraces")
for file in glob.glob(os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')):
if exists(file):
getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file))
os.remove(file)
cmdline = xharnesscommand() + ['android', 'state', '--adb']
adb = RunCommand(cmdline, verbose=True)
adb.run()
# Do not remove, XHarness install seems to fail without an adb command called before the xharness command
getLogger().info("Preparing ADB")
cmdline = [
adb.stdout.strip(),
'shell',
'wm',
'size'
]
RunCommand(cmdline, verbose=True).run()
# Get animation values
getLogger().info("Getting Values we will need set specifically")
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'global', 'window_animation_scale'
]
window_animation_scale_cmd = RunCommand(cmdline, verbose=True)
window_animation_scale_cmd.run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'global', 'transition_animation_scale'
]
transition_animation_scale_cmd = RunCommand(cmdline, verbose=True)
transition_animation_scale_cmd.run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'global', 'animator_duration_scale'
]
animator_duration_scale_cmd = RunCommand(cmdline, verbose=True)
animator_duration_scale_cmd.run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'system', 'screen_off_timeout'
]
screen_off_timeout_cmd = RunCommand(cmdline, verbose=True)
screen_off_timeout_cmd.run()
getLogger().info(f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}, screen timeout {screen_off_timeout_cmd.stdout.strip()}")
# Make sure animations are set to 1 or disabled
getLogger().info("Setting needed values")
if(self.animationsdisabled):
animationValue = 0
else:
animationValue = 1
minimumTimeoutValue = 2 * 60 * 1000 # milliseconds
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'global', 'transition_animation_scale', str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'global', 'animator_duration_scale', str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'system', 'screen_off_timeout', str(minimumTimeoutValue)
]
if(minimumTimeoutValue > int(screen_off_timeout_cmd.stdout.strip())):
getLogger().info("Screen off value is lower than minimum time, setting to higher time")
RunCommand(cmdline, verbose=True).run()
# Check for success
getLogger().info("Getting animation values to verify it worked")
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'global', 'window_animation_scale'
]
windowSetValue = RunCommand(cmdline, verbose=True)
windowSetValue.run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'global', 'transition_animation_scale'
]
transitionSetValue = RunCommand(cmdline, verbose=True)
transitionSetValue.run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'get', 'global', 'animator_duration_scale'
]
animatorSetValue = RunCommand(cmdline, verbose=True)
animatorSetValue.run()
if(int(windowSetValue.stdout.strip()) != animationValue or int(transitionSetValue.stdout.strip()) != animationValue or int(animatorSetValue.stdout.strip()) != animationValue):
# Setting the values didn't work, error out
getLogger().exception(f"Failed to set animation values to {animationValue}.")
raise Exception(f"Failed to set animation values to {animationValue}.")
else:
getLogger().info(f"Animation values successfully set to {animationValue}.")
try:
installCmd = xharnesscommand() + [
'android',
'install',
'--app', self.packagepath,
'--package-name',
self.packagename,
'-o',
const.TRACEDIR,
'-v'
]
RunCommand(installCmd, verbose=True).run()
getLogger().info("Completed install, running shell.")
cmdline = [
adb.stdout.strip(),
'shell',
f'cmd package resolve-activity --brief {self.packagename} | tail -n 1'
]
getActivity = RunCommand(cmdline, verbose=True)
getActivity.run()
getLogger().info(f"Target Activity {getActivity.stdout}")
# More setup stuff
checkScreenOnCmd = [
adb.stdout.strip(),
'shell',
f'dumpsys input_method | grep mInteractive'
]
checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True)
checkScreenOn.run()
keyInputCmd = [
adb.stdout.strip(),
'shell',
'input',
'keyevent'
]
if("mInteractive=false" in checkScreenOn.stdout):
# Turn on the screen to make interactive and see if it worked
getLogger().info("Screen was off, turning on.")
screenWasOff = True
RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key
RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock the screen with menu key (only works if it is not a password lock)
checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True)
checkScreenOn.run()
if("mInteractive=false" in checkScreenOn.stdout):
getLogger().exception("Failed to make screen interactive.")
raise Exception("Failed to make screen interactive.")
# Actual testing some run stuff
getLogger().info("Test run to check if permissions are needed")
activityname = getActivity.stdout
# -W in the start command waits for the app to finish initial draw.
startAppCmd = [
adb.stdout.strip(),
'shell',
'am',
'start-activity',
'-W',
'-n',
activityname
]
testRun = RunCommand(startAppCmd, verbose=True)
testRun.run()
testRunStats = re.findall(runSplitRegex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime)
getLogger().info(f"Test run activity: {testRunStats[3]}")
time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle
stopAppCmd = [
adb.stdout.strip(),
'shell',
'am',
'force-stop',
self.packagename
]
RunCommand(stopAppCmd, verbose=True).run()
if "com.google.android.permissioncontroller" in testRunStats[3]:
# On perm screen, use the buttons to close it. it will stay away until the app is reinstalled
RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen
time.sleep(1)
RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen
time.sleep(1)
# Check to make sure it worked
testRun = RunCommand(startAppCmd, verbose=True)
testRun.run()
testRunStats = re.findall(runSplitRegex, testRun.stdout)
getLogger().info(f"Test run activity: {testRunStats[3]}")
RunCommand(stopAppCmd, verbose=True).run()
if "com.google.android.permissioncontroller" in testRunStats[3]:
getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.")
raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.")
# Create the fullydrawn command
fullyDrawnRetrieveCmd = [
adb.stdout.strip(),
'shell',
f"logcat -d | grep 'ActivityTaskManager: Fully drawn {self.packagename}'"
]
basicStartupRetrieveCmd = [
adb.stdout.strip(),
'shell',
f"logcat -d | grep 'ActivityTaskManager: Displayed {activityname}'"
]
clearLogsCmd = [
adb.stdout.strip(),
'logcat',
'-c'
]
allResults = []
for i in range(self.startupiterations):
# Clear logs
RunCommand(clearLogsCmd, verbose=True).run()
startStats = RunCommand(startAppCmd, verbose=True)
startStats.run()
# Make sure we cold started (TODO Add other starts)
if("LaunchState: COLD" not in startStats.stdout):
getLogger().error("App Start not COLD!")
# Save the results and get them from the log
if self.usefullydrawntime: time.sleep(self.fullyDrawnDelaySecMax) # Start command doesn't wait for fully drawn report, force a wait for it. -W in the start command waits for the app to finish initial draw.
RunCommand(stopAppCmd, verbose=True).run()
if self.usefullydrawntime:
retrieveTimeCmd = RunCommand(fullyDrawnRetrieveCmd, verbose=True)
else:
retrieveTimeCmd = RunCommand(basicStartupRetrieveCmd, verbose=True)
retrieveTimeCmd.run()
dirtyCapture = re.search("\+(\d*s?\d+)ms", retrieveTimeCmd.stdout)
if not dirtyCapture:
raise Exception("Failed to capture the reported start time!")
formattedTime = f"TotalTime: {dirtyCapture.group(1).replace('s', '')}\n"
allResults.append(formattedTime) # append TotalTime: (TIME)
time.sleep(3) # Delay in seconds for ensuring a cold start
finally:
getLogger().info("Stopping App for uninstall")
RunCommand(stopAppCmd, verbose=True).run()
getLogger().info("Uninstalling app")
uninstallAppCmd = xharnesscommand() + [
'android',
'uninstall',
'--package-name',
self.packagename
]
RunCommand(uninstallAppCmd, verbose=True).run()
# Reset animation values
getLogger().info("Resetting animation values to pretest values")
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'global', 'window_animation_scale', window_animation_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'global', 'transition_animation_scale', transition_animation_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'global', 'animator_duration_scale', animator_duration_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(),
'shell', 'settings', 'put', 'system', 'screen_off_timeout', screen_off_timeout_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
if screenWasOff:
RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off
# Create traces to store the data so we can keep the current general parse trace flow
getLogger().info(f"Logs: \n{allResults}")
os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True)
traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w")
for result in allResults:
traceFile.write(result)
traceFile.close()
startup = StartupWrapper()
self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname)
startup.parsetraces(self.traits)
elif self.testtype == const.DEVICESTARTUP and self.devicetype == 'ios':
getLogger().info("Clearing potential previous run nettraces")
for file in glob.glob(os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')):
if exists(file):
getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file))
os.remove(file)
if not exists(const.TMPDIR):
os.mkdir(const.TMPDIR)
getLogger().info("Clearing potential previous run *.logarchive")
for logarchive in glob.glob(os.path.join(const.TMPDIR, '*.logarchive')):
if exists(logarchive):
getLogger().info("Removed: " + os.path.join(const.TMPDIR, logarchive))
rmtree(logarchive)
getLogger().info("Checking device state.")
cmdline = xharnesscommand() + ['apple', 'state']
adb = RunCommand(cmdline, verbose=True)
adb.run()
getLogger().info("Installing app on device.")
installCmd = xharnesscommand() + [
'apple',
'install',
'--app', self.packagepath,
'--target', 'ios-device',
'-o',
const.TRACEDIR,
'-v'
]
RunCommand(installCmd, verbose=True).run()
getLogger().info("Completed install.")
allResults = []
for i in range(self.startupiterations + 1): # adding one iteration to account for the warmup iteration
getLogger().info("Waiting 10 secs to ensure we're not getting confused with previous app run.")
time.sleep(10)
getLogger().info(f"Collect startup data for iteration {i}.")
runCmdTimestamp = datetime.now()
runCmd = xharnesscommand() + [
'apple',
'mlaunch',
'--',
f'--launchdevbundleid={self.packagename}',
]
runCmdCommand = RunCommand(runCmd, verbose=True)
runCmdCommand.run()
app_pid_search = re.search("Launched application.*with pid (?P<app_pid>\d+)", runCmdCommand.stdout)
app_pid = int(app_pid_search.group('app_pid'))
logarchive_filename = os.path.join(const.TMPDIR, f'iteration{i}.logarchive')
getLogger().info(f"Waiting 5 secs to ensure app with PID {app_pid} is fully started.")
time.sleep(5)
collectCmd = [
'sudo',
'log',
'collect',
'--device',
'--start', runCmdTimestamp.strftime("%Y-%m-%d %H:%M:%S"),
'--output', logarchive_filename,
]
RunCommand(collectCmd, verbose=True).run()
getLogger().info(f"Kill app with PID {app_pid}.")
killCmd = xharnesscommand() + [
'apple',
'mlaunch',
'--',
f'--killdev={app_pid}',
]
killCmdCommand = RunCommand(killCmd, verbose=True)
killCmdCommand.run()
# Process Data
# There are four watchdog events from SpringBoard during an application startup:
#
# [application<net.dot.maui>:770] [realTime] Now monitoring resource allowance of 20.00s (at refreshInterval -1.00s)
# [application<net.dot.maui>:770] [realTime] Stopped monitoring.
# [application<net.dot.maui>:770] [realTime] Now monitoring resource allowance of 19.28s (at refreshInterval -1.00s)
# [application<net.dot.maui>:770] [realTime] Stopped monitoring.
#
# The first two are monitoring the time it takes the OS to create the process, load .dylibs and call into the app's main()
# The second two are monitoring the time it takes the app to draw the first frame of UI from main()
#
# An app has 20 seconds to complete this sequence or the OS will kill the app.
# We collect these log events to do our measurements.
logShowCmd = [
'log',
'show',
'--predicate', '(process == "SpringBoard") && (category == "Watchdog")',
'--info',
'--style', 'ndjson',
logarchive_filename,
]
logShowCmdCommand = RunCommand(logShowCmd, verbose=True)
logShowCmdCommand.run()
events = []
for line in logShowCmdCommand.stdout.splitlines():
try:
lineData = json.loads(line)
if 'Now monitoring resource allowance' in lineData['eventMessage'] or 'Stopped monitoring' in lineData['eventMessage']:
events.append (lineData)
except:
break
# the startup measurement relies on the date/time of the device to be pretty much in sync with the host
# since we use the timestamps from the host to decide which parts of the device log to get and
# we then use that to calculate the time delta from watchdog events
if len(events) != 4:
raise Exception("Didn't get the right amount of watchdog events, this could mean the app crashed or the device clock is not in sync with the host.")
timeToMainEventStart = events[0]
timeToMainEventStop = events[1]
timeToFirstDrawEventStart = events[2]
timeToFirstDrawEventStop = events[3]
# validate log messages
if f'application<{self.packagename}>:{app_pid}' not in timeToMainEventStart['eventMessage'] or 'Now monitoring resource allowance of 20.00s' not in timeToMainEventStart['eventMessage']:
raise Exception(f"Invalid timeToMainEventStart: {timeToMainEventStart['eventMessage']}")
if f'application<{self.packagename}>:{app_pid}' not in timeToMainEventStop['eventMessage'] or 'Stopped monitoring' not in timeToMainEventStop['eventMessage']:
raise Exception(f"Invalid timeToMainEventStop: {timeToMainEventStop['eventMessage']}")
if f'application<{self.packagename}>:{app_pid}' not in timeToFirstDrawEventStart['eventMessage'] or 'Now monitoring resource allowance of' not in timeToFirstDrawEventStart['eventMessage']:
raise Exception(f"Invalid timeToFirstDrawEventStart: {timeToFirstDrawEventStart['eventMessage']}")
if f'application<{self.packagename}>:{app_pid}' not in timeToFirstDrawEventStop['eventMessage'] or 'Stopped monitoring' not in timeToFirstDrawEventStop['eventMessage']:
raise Exception(f"Invalid timeToFirstDrawEventStop: {timeToFirstDrawEventStop['eventMessage']}")
timeToMainEventStartDateTime = datetime.strptime(timeToMainEventStart['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z')
timeToMainEventEndDateTime = datetime.strptime(timeToMainEventStop['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z')
timeToMainMilliseconds = (timeToMainEventEndDateTime - timeToMainEventStartDateTime).total_seconds() * 1000
timeToFirstDrawEventStartDateTime = datetime.strptime(timeToFirstDrawEventStart['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z')
timeToFirstDrawEventEndDateTime = datetime.strptime(timeToFirstDrawEventStop['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z')
timeToFirstDrawMilliseconds = (timeToFirstDrawEventEndDateTime - timeToFirstDrawEventStartDateTime).total_seconds() * 1000
if self.usefullydrawntime:
# grab log event with the magic string in it
logShowMagicStringCmd = [
'log',
'show',
'--predicate', f'(processIdentifier == {app_pid}) && (composedMessage contains "{self.fullyDrawnMagicString}")',
'--info',
'--style', 'ndjson',
logarchive_filename,
]
logShowMagicStringCmd = RunCommand(logShowMagicStringCmd, verbose=True)
logShowMagicStringCmd.run()
magicStringEvent = ''
for line in logShowMagicStringCmd.stdout.splitlines():
try:
lineData = json.loads(line)
if self.fullyDrawnMagicString in lineData['eventMessage']:
magicStringEvent = lineData
except:
break
if magicStringEvent == '':
raise Exception("Didn't get the fully-drawn magic string event.")
timeToMagicStringEventDateTime = datetime.strptime(magicStringEvent['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z')
# startup time is time to the magic string event
totalTimeMilliseconds = (timeToMagicStringEventDateTime - timeToMainEventStartDateTime).total_seconds() * 1000
else:
# startup time is time to first draw
totalTimeMilliseconds = timeToMainMilliseconds + timeToFirstDrawMilliseconds
if i == 0:
# ignore the warmup iteration
getLogger().info(f'Warmup iteration took {totalTimeMilliseconds}')
else:
# TODO: this isn't really a COLD run, we should have separate measurements for starting the app right after install
launchState = 'COLD'
allResults.append(f'LaunchState: {launchState}\nTotalTime: {int(totalTimeMilliseconds)}\nTimeToMain: {int(timeToMainMilliseconds)}\n\n')
# Done with testing, uninstall the app
getLogger().info("Uninstalling app")
uninstallAppCmd = xharnesscommand() + [
'apple',
'uninstall',
'--app', self.packagename,
'--target', 'ios-device',
'-o',
const.TRACEDIR,
'-v'
]
RunCommand(uninstallAppCmd, verbose=True).run()
# Create traces to store the data so we can keep the current general parse trace flow
getLogger().info(f"Logs: \n{allResults}")
os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True)
traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w")
for result in allResults:
traceFile.write(result)
traceFile.close()
startup = StartupWrapper()
self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname)
startup.parsetraces(self.traits)
elif self.testtype == const.SOD:
sod = SODWrapper()
builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None
if not builtdir:
builtdir = const.BINDIR if os.path.exists(const.BINDIR) else None
if not (self.dirs or builtdir):
raise Exception("Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR))
sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)
class PreCommands:
'''
Handles building and publishing
'''
def __init__(self):
self.project: CSharpProject
self.projectfile: CSharpProjFile
self.crossgen_arguments = CrossgenArguments()
parser = ArgumentParser()
subparsers = parser.add_subparsers(
title='Operations',
description=
'Common preperation steps for perf tests. Should run under src\scenarios\<test asset folder>',
dest='operation')
default_parser = subparsers.add_parser(
DEFAULT,
help=
'Default operation (placeholder command and no specific operation will be executed)'
)
self.add_common_arguments(default_parser)
build_parser = subparsers.add_parser(BUILD, help='Builds the project')
self.add_common_arguments(build_parser)
publish_parser = subparsers.add_parser(PUBLISH,
help='Publishes the project')
self.add_common_arguments(publish_parser)
crossgen_parser = subparsers.add_parser(
CROSSGEN, help='Runs crossgen on a particular file')
self.add_common_arguments(crossgen_parser)
self.crossgen_arguments.add_crossgen_arguments(crossgen_parser)
crossgen2_parser = subparsers.add_parser(
CROSSGEN2, help='Runs crossgen2 on a particular file')
self.add_common_arguments(crossgen2_parser)
self.crossgen_arguments.add_crossgen2_arguments(crossgen2_parser)
args = parser.parse_args()
if not args.operation:
getLogger().error("Please specify an operation: %s" %
list(OPERATIONS))
sys.exit(1)
self.configuration = args.configuration
self.operation = args.operation
self.framework = args.framework
self.runtime_identifier = args.runtime
self.msbuild = args.msbuild
self.msbuildstatic = args.msbuildstatic
self.binlog = args.binlog
if self.operation == CROSSGEN:
self.crossgen_arguments.parse_crossgen_args(args)
if self.operation == CROSSGEN2:
self.crossgen_arguments.parse_crossgen2_args(args)
def new(self,
template: str,
output_dir: str,
bin_dir: str,
exename: str,
working_directory: str,
language: str = None):
'makes a new app with the given template'
self.project = CSharpProject.new(template=template,
output_dir=output_dir,
bin_dir=bin_dir,
exename=exename,
working_directory=working_directory,
force=True,
verbose=True,
language=language)
self._updateframework(self.project.csproj_file)
self._addstaticmsbuildproperty(self.project.csproj_file)
def add_common_arguments(self, parser: ArgumentParser):
"Options that are common across many 'dotnet' commands"
parser.add_argument(
'-c',
'--configuration',
dest='configuration',
choices=[DEBUG, RELEASE],
metavar='config',
help='configuration for build or publish - ex: Release or Debug')
parser.add_argument(
'-f',
'--framework',
dest='framework',
metavar='framework',
help='framework for build or publish - ex: netcoreapp3.0')
parser.add_argument('-r',
'--runtime',
dest='runtime',
metavar='runtime',
help='runtime for build or publish - ex: win-x64')
parser.add_argument(
'--msbuild',
dest='msbuild',
metavar='msbuild',
help=
'a list of msbuild flags passed to build or publish command separated by semicolons - ex: /p:Foo=Bar;/p:Baz=Blee;...'
)
parser.add_argument(
'--msbuild-static',
dest='msbuildstatic',
metavar='msbuildstatic',
help=
'a list of msbuild properties inserted into .csproj file of the project - ex: Foo=Bar;Bas=Blee;'
)
parser.add_argument(
'--binlog',
dest='binlog',
metavar='<file-name>.binlog',
help=
'flag to turn on binlog for build or publish; ex: <file-name>.binlog'
)
parser.set_defaults(configuration=RELEASE)
def existing(self, projectdir: str, projectfile: str):
'create a project from existing project file'
self._backup(projectdir)
csproj = CSharpProjFile(os.path.join(const.APPDIR, projectfile),
sys.path[0])
self.project = CSharpProject(csproj, const.BINDIR)
self._updateframework(csproj.file_name)
def execute(self):
'Parses args and runs precommands'
if self.operation == DEFAULT:
pass
if self.operation == BUILD:
self._restore()
self._build(configuration=self.configuration,
framework=self.framework)
if self.operation == PUBLISH:
self._restore()
self._publish(configuration=self.configuration,
runtime_identifier=self.runtime_identifier)
if self.operation == CROSSGEN:
startup_args = [
os.path.join(self.crossgen_arguments.coreroot,
'crossgen%s' % extension()),
]
startup_args += self.crossgen_arguments.get_crossgen_command_line()
RunCommand(startup_args,
verbose=True).run(self.crossgen_arguments.coreroot)
if self.operation == CROSSGEN2:
startup_args = [
os.path.join(self.crossgen_arguments.coreroot,
'corerun%s' % extension()),
os.path.join(self.crossgen_arguments.coreroot, 'crossgen2',
'crossgen2.dll'),
]
startup_args += self.crossgen_arguments.get_crossgen2_command_line(
)
RunCommand(startup_args,
verbose=True).run(self.crossgen_arguments.coreroot)
def add_startup_logging(self, file: str, line: str):
self.add_event_source(file, line,
"PerfLabGenericEventSource.Log.Startup();")
def add_event_source(self, file: str, line: str, trace_statement: str):
'''
Adds a copy of the event source to the project and inserts the correct call
file: relative path to the root of the project (where the project file lives)
line: Exact line to insert trace statement after
trace_statement: Statement to insert
'''
projpath = os.path.dirname(self.project.csproj_file)
staticpath = os.path.join(get_repo_root_path(), "src", "scenarios",
"staticdeps")
if helixpayload():
staticpath = os.path.join(helixpayload(), "staticdeps")
shutil.copyfile(os.path.join(staticpath, "PerfLab.cs"),
os.path.join(projpath, "PerfLab.cs"))
filepath = os.path.join(projpath, file)
insert_after(filepath, line, trace_statement)
def _addstaticmsbuildproperty(self, projectfile: str):
'Insert static msbuild property in the specified project file'
if self.msbuildstatic:
for propertyarg in self.msbuildstatic.split(';'):
propertyname, propertyvalue = propertyarg.split('=')
propertystring = f'\n <PropertyGroup>\n <{propertyname}>{propertyvalue}</{propertyname}>\n </PropertyGroup>'
insert_after(projectfile, r'</PropertyGroup>', propertystring)
def _updateframework(self, projectfile: str):
'Update the <TargetFramework> property so we can re-use the template'
if self.framework:
replace_line(
projectfile, r'<TargetFramework>.*?</TargetFramework>',
f'<TargetFramework>{self.framework}</TargetFramework>')
def _publish(self,
configuration: str,
framework: str = None,
runtime_identifier: str = None):
self.project.publish(
configuration,
const.PUBDIR,
True,
os.path.join(
get_packages_directory(), ''
), # blazor publish targets require the trailing slash for joining the paths
framework,
runtime_identifier,
self.msbuild or "",
'-bl:%s' % self.binlog if self.binlog else "")
def _restore(self):
self.project.restore(packages_path=get_packages_directory(),
verbose=True)
def _build(self, configuration: str, framework: str = None):
self.project.build(configuration=configuration,
verbose=True,
packages_path=get_packages_directory(),
target_framework_monikers=[framework],
output_to_bindir=True)
def _backup(self, projectdir: str):
'Copy from projectdir to appdir so we do not modify the source code'
if os.path.isdir(const.APPDIR):
shutil.rmtree(const.APPDIR)
shutil.copytree(projectdir, const.APPDIR)
class Runner:
'''
Wrapper for running all the things
'''
def __init__(self, traits: TestTraits):
self.traits = traits
self.testtype = None
self.sdktype = None
self.scenarioname = None
self.coreroot = None
self.crossgenfile = None
self.dirs = None
self.crossgen_arguments = CrossgenArguments()
setup_loggers(True)
def parseargs(self):
'''
Parses input args to the script
'''
parser = ArgumentParser(
description=
'test.py runs the test with specified commands. Usage: test.py <command> <optional subcommands> <options>',
formatter_class=RawTextHelpFormatter)
subparsers = parser.add_subparsers(
title='subcommands for scenario tests', dest='testtype')
# startup command
startupparser = subparsers.add_parser(
const.STARTUP,
description='measure time to main of running the project')
self.add_common_arguments(startupparser)
# parse only command
parseonlyparser = subparsers.add_parser(
const.DEVICESTARTUP,
description='measure time to main for Android apps')
parseonlyparser.add_argument('--device-type',
choices=['android', 'ios'],
type=str.lower,
help='Device type for testing',
dest='devicetype')
parseonlyparser.add_argument('--package-path',
help='Location of test application',
dest='packagepath')
parseonlyparser.add_argument('--package-name',
help='Classname of application',
dest='packagename')
parseonlyparser.add_argument('--startup-iterations',
help='Startups to run (1+)',
type=int,
default=5,
dest='startupiterations')
parseonlyparser.add_argument('--disable-animations',
help='Disable Android device animations',
action='store_true',
dest='animationsdisabled')
self.add_common_arguments(parseonlyparser)
# inner loop command
innerloopparser = subparsers.add_parser(
const.INNERLOOP,
description=
'measure time to main and difference between two runs in a row')
self.add_common_arguments(innerloopparser)
# inner loop msbuild command
innerloopparser = subparsers.add_parser(
const.INNERLOOPMSBUILD,
description=
'measure time to main and difference between two runs in a row')
self.add_common_arguments(innerloopparser)
# dotnet watch command
dotnetwatchparser = subparsers.add_parser(
const.DOTNETWATCH,
description='measure time to main and time for hot reload')
self.add_common_arguments(dotnetwatchparser)
# sdk command
sdkparser = subparsers.add_parser(
const.SDK,
description='subcommands for sdk scenario',
formatter_class=RawTextHelpFormatter)
sdkparser.add_argument('sdktype',
choices=[
const.CLEAN_BUILD, const.BUILD_NO_CHANGE,
const.NEW_CONSOLE
],
type=str.lower,
help='''
clean_build: measure duration of building from source in each iteration
build_no_change: measure duration of building with existing output in each iteration
new_console: measure duration of creating a new console template
''')
self.add_common_arguments(sdkparser)
crossgenparser = subparsers.add_parser(
const.CROSSGEN,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen_arguments(crossgenparser)
self.add_common_arguments(crossgenparser)
crossgen2parser = subparsers.add_parser(
const.CROSSGEN2,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen2_arguments(crossgen2parser)
self.add_common_arguments(crossgen2parser)
sodparser = subparsers.add_parser(
const.SOD,
description=
'measure size on disk of the specified directory and its children')
sodparser.add_argument('--dirs',
dest='dirs',
type=str,
help=r'''
directories to measure separated by semicolon
ex: C:\repos\performance;C:\repos\runtime
''')
self.add_common_arguments(sodparser)
args = parser.parse_args()
if not args.testtype:
getLogger().error(
"Please specify a test type: %s. Type test.py <test type> -- help for more type-specific subcommands"
% testtypes)
sys.exit(1)
self.testtype = args.testtype
if self.testtype == const.SDK:
self.sdktype = args.sdktype
if self.testtype == const.CROSSGEN:
self.crossgen_arguments.parse_crossgen_args(args)
if self.testtype == const.CROSSGEN2:
self.crossgen_arguments.parse_crossgen2_args(args)
if self.testtype == const.SOD:
self.dirs = args.dirs
if self.testtype == const.DEVICESTARTUP:
self.packagepath = args.packagepath
self.packagename = args.packagename
self.devicetype = args.devicetype
self.startupiterations = args.startupiterations
self.animationsdisabled = args.animationsdisabled
if args.scenarioname:
self.scenarioname = args.scenarioname
def add_common_arguments(self, parser: ArgumentParser):
"Common arguments to add to subparsers"
parser.add_argument('--scenario-name', dest='scenarioname')
def run(self):
'''
Runs the specified scenario
'''
self.parseargs()
if self.testtype == const.INNERLOOP:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.INNERLOOPMSBUILD:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.DOTNETWATCH:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH],
apptorun='dotnet',
appargs='watch -v',
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
self.traits.add_traits(workingdir=const.APPDIR)
startup.runtests(self.traits)
if self.testtype == const.STARTUP:
startup = StartupWrapper()
self.traits.add_traits(
overwrite=False,
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '',
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.STARTUP],
apptorun=publishedexe(self.traits.exename),
)
startup.runtests(self.traits)
elif self.testtype == const.SDK:
startup = StartupWrapper()
envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0'
envlistcleanbuild = ';'.join(
['MSBUILDDISABLENODEREUSE=1', envlistbuild])
# clean build
if self.sdktype == const.CLEAN_BUILD:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
apptorun=const.DOTNET,
appargs='build',
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR,
environmentvariables=envlistcleanbuild,
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# build(no changes)
if self.sdktype == const.BUILD_NO_CHANGE:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE),
apptorun=const.DOTNET,
appargs='build',
workingdir=const.APPDIR,
environmentvariables=envlistbuild)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# new console
if self.sdktype == const.NEW_CONSOLE:
self.traits.add_traits(
overwrite=False,
appargs='new console',
apptorun=const.DOTNET,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_new' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN:
startup = StartupWrapper()
crossgenexe = 'crossgen%s' % extension()
crossgenargs = self.crossgen_arguments.get_crossgen_command_line()
coreroot = self.crossgen_arguments.coreroot
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME,
workingdir=coreroot,
appargs=' '.join(crossgenargs))
self.traits.add_traits(
overwrite=False,
scenarioname='Crossgen Throughput - %s' % scenario_filename,
scenariotypename='%s - %s' %
(const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename),
apptorun='%s\%s' % (coreroot, crossgenexe),
)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN2:
startup = StartupWrapper()
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
crossgen2args = self.crossgen_arguments.get_crossgen2_command_line(
)
compiletype = self.crossgen_arguments.crossgen2_compiletype()
scenarioname = 'Crossgen2 Throughput - %s - %s' % (
compiletype, scenario_filename)
if self.crossgen_arguments.singlethreaded:
scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (
compiletype, scenario_filename)
if compiletype == const.CROSSGEN2_COMPOSITE:
self.traits.add_traits(overwrite=True, skipprofile='true')
self.traits.add_traits(
overwrite=True,
startupmetric=const.STARTUP_CROSSGEN2,
workingdir=self.crossgen_arguments.coreroot,
appargs='%s %s' % (os.path.join(
'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args)))
self.traits.add_traits(
overwrite=False,
scenarioname=scenarioname,
apptorun=os.path.join(self.crossgen_arguments.coreroot,
'corerun%s' % extension()),
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '' # turn on clr user events
)
startup.runtests(self.traits)
elif self.testtype == const.DEVICESTARTUP:
# ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc
# Regex used to split the response from starting the activity and saving each value
#Example:
# Starting: Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }
# Status: ok
# LaunchState: COLD
# Activity: net.dot.HelloAndroid/net.dot.MainActivity
# TotalTime: 241
# WaitTime: 242
# Complete
# Saves: [Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }, ok, COLD, net.dot.HelloAndroid/net.dot.MainActivity, 241, 242]
# Split results (start at 0) (List is Starting (Intent activity), Status (ok...), LaunchState ([HOT, COLD, WARM]), Activity (started activity name), TotalTime(toFrameOne), WaitTime(toFullLoad))
runSplitRegex = ":\s(.+)"
screenWasOff = False
getLogger().info("Clearing potential previous run nettraces")
for file in glob.glob(
os.path.join(const.TRACEDIR, 'PerfTest',
'runoutput.trace')):
if exists(file):
getLogger().info("Removed: " +
os.path.join(const.TRACEDIR, file))
os.remove(file)
cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb']
adb = RunCommand(cmdline, verbose=True)
adb.run()
# Do not remove, XHarness install seems to fail without an adb command called before the xharness command
getLogger().info("Preparing ADB")
cmdline = [adb.stdout.strip(), 'shell', 'wm', 'size']
RunCommand(cmdline, verbose=True).run()
# Get animation values
getLogger().info("Getting animation values")
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'window_animation_scale'
]
window_animation_scale_cmd = RunCommand(cmdline, verbose=True)
window_animation_scale_cmd.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'transition_animation_scale'
]
transition_animation_scale_cmd = RunCommand(cmdline, verbose=True)
transition_animation_scale_cmd.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'animator_duration_scale'
]
animator_duration_scale_cmd = RunCommand(cmdline, verbose=True)
animator_duration_scale_cmd.run()
getLogger().info(
f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}"
)
# Make sure animations are set to 1 or disabled
getLogger().info("Setting animation values")
if (self.animationsdisabled):
animationValue = 0
else:
animationValue = 1
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'window_animation_scale',
str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'transition_animation_scale',
str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'animator_duration_scale',
str(animationValue)
]
RunCommand(cmdline, verbose=True).run()
# Check for success
getLogger().info("Getting animation values to verify it worked")
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'window_animation_scale'
]
windowSetValue = RunCommand(cmdline, verbose=True)
windowSetValue.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'transition_animation_scale'
]
transitionSetValue = RunCommand(cmdline, verbose=True)
transitionSetValue.run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'get', 'global',
'animator_duration_scale'
]
animatorSetValue = RunCommand(cmdline, verbose=True)
animatorSetValue.run()
if (int(windowSetValue.stdout.strip()) != animationValue
or int(transitionSetValue.stdout.strip()) != animationValue
or int(animatorSetValue.stdout.strip()) != animationValue):
# Setting the values didn't work, error out
getLogger().exception(
f"Failed to set animation values to {animationValue}.")
exit(-1)
else:
getLogger().info(
f"Animation values successfully set to {animationValue}.")
installCmd = xharnesscommand() + [
self.devicetype, 'install', '--app', self.packagepath,
'--package-name', self.packagename, '-o', const.TRACEDIR, '-v'
]
RunCommand(installCmd, verbose=True).run()
getLogger().info("Completed install, running shell.")
cmdline = [
adb.stdout.strip(), 'shell',
f'cmd package resolve-activity --brief {self.packagename} | tail -n 1'
]
getActivity = RunCommand(cmdline, verbose=True)
getActivity.run()
getLogger().info(f"Target Activity {getActivity.stdout}")
# More setup stuff
checkScreenOnCmd = [
adb.stdout.strip(), 'shell',
f'dumpsys input_method | grep mInteractive'
]
checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True)
checkScreenOn.run()
keyInputCmd = [adb.stdout.strip(), 'shell', 'input', 'keyevent']
if ("mInteractive=false" in checkScreenOn.stdout):
# Turn on the screen to make interactive and see if it worked
getLogger().info("Screen was off, turning on.")
screenWasOff = True
RunCommand(keyInputCmd + ['26'],
verbose=True).run() # Press the power key
RunCommand(keyInputCmd + ['82'], verbose=True).run(
) # Unlock the screen with menu key (only works if it is not a password lock)
checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True)
checkScreenOn.run()
if ("mInteractive=false" in checkScreenOn.stdout):
getLogger().exception("Failed to make screen interactive.")
exit(-1)
# Actual testing some run stuff
getLogger().info("Test run to check if permissions are needed")
activityname = getActivity.stdout
startAppCmd = [
adb.stdout.strip(), 'shell', 'am', 'start-activity', '-W',
'-n', activityname
]
testRun = RunCommand(startAppCmd, verbose=True)
testRun.run()
testRunStats = re.findall(
runSplitRegex, testRun.stdout
) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime)
getLogger().info(f"Test run activity: {testRunStats[3]}")
stopAppCmd = [
adb.stdout.strip(), 'shell', 'am', 'force-stop',
self.packagename
]
RunCommand(stopAppCmd, verbose=True).run()
if "com.google.android.permissioncontroller" in testRunStats[3]:
# On perm screen, use the buttons to close it. it will stay away until the app is reinstalled
RunCommand(keyInputCmd + ['22'],
verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['22'],
verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['66'], verbose=True).run(
) # Press enter to close main perm screen
time.sleep(1)
RunCommand(keyInputCmd + ['22'],
verbose=True).run() # Select next button
time.sleep(1)
RunCommand(keyInputCmd + ['66'], verbose=True).run(
) # Press enter to close out of second screen
time.sleep(1)
# Check to make sure it worked
testRun = RunCommand(startAppCmd, verbose=True)
testRun.run()
testRunStats = re.findall(runSplitRegex, testRun.stdout)
getLogger().info(f"Test run activity: {testRunStats[3]}")
RunCommand(stopAppCmd, verbose=True).run()
if "com.google.android.permissioncontroller" in testRunStats[
3]:
getLogger().exception(
"Failed to get past permission screen, run locally to see if enough next button presses were used."
)
exit(-1)
allResults = []
for i in range(self.startupiterations):
startStats = RunCommand(startAppCmd, verbose=True)
startStats.run()
RunCommand(stopAppCmd, verbose=True).run()
allResults.append(
startStats.stdout
) # Save results (List is Intent, Status, LaunchState Activity, TotalTime, WaitTime)
time.sleep(3) # Delay in seconds for ensuring a cold start
getLogger().info("Stopping App for uninstall")
RunCommand(stopAppCmd, verbose=True).run()
getLogger().info("Uninstalling app")
uninstallAppCmd = xharnesscommand() + [
'android', 'uninstall', '--package-name', self.packagename
]
RunCommand(uninstallAppCmd, verbose=True).run()
# Reset animation values
getLogger().info("Resetting animation values to pretest values")
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'window_animation_scale',
window_animation_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'transition_animation_scale',
transition_animation_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'settings', 'put', 'global',
'animator_duration_scale',
animator_duration_scale_cmd.stdout.strip()
]
RunCommand(cmdline, verbose=True).run()
if screenWasOff:
RunCommand(keyInputCmd + ['26'],
verbose=True).run() # Turn the screen back off
# Create traces to store the data so we can keep the current general parse trace flow
getLogger().info(f"Logs: \n{allResults}")
os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True)
traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w")
for result in allResults:
traceFile.write(result)
traceFile.close()
startup = StartupWrapper()
self.traits.add_traits(
overwrite=True,
apptorun="app",
startupmetric=const.STARTUP_DEVICETIMETOMAIN,
tracefolder='PerfTest/',
tracename='runoutput.trace',
scenarioname=self.scenarioname)
startup.parsetraces(self.traits)
elif self.testtype == const.SOD:
sod = SODWrapper()
builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None
if not builtdir:
builtdir = const.BINDIR if os.path.exists(
const.BINDIR) else None
if not (self.dirs or builtdir):
raise Exception(
"Dirs was not passed in and neither %s nor %s exist" %
(const.PUBDIR, const.BINDIR))
sod.runtests(scenarioname=self.scenarioname,
dirs=self.dirs or builtdir,
artifact=self.traits.artifact)
class Runner:
'''
Wrapper for running all the things
'''
def __init__(self, traits: TestTraits):
self.traits = traits
self.testtype = None
self.sdktype = None
self.scenarioname = None
self.coreroot = None
self.crossgenfile = None
self.dirs = None
self.crossgen_arguments = CrossgenArguments()
setup_loggers(True)
def parseargs(self):
'''
Parses input args to the script
'''
parser = ArgumentParser(
description=
'test.py runs the test with specified commands. Usage: test.py <command> <optional subcommands> <options>',
formatter_class=RawTextHelpFormatter)
subparsers = parser.add_subparsers(
title='subcommands for scenario tests', dest='testtype')
# startup command
startupparser = subparsers.add_parser(
const.STARTUP,
description='measure time to main of running the project')
self.add_common_arguments(startupparser)
# parse only command
parseonlyparser = subparsers.add_parser(
const.DEVICESTARTUP,
description='measure time to main for Android apps')
parseonlyparser.add_argument('--device-type',
choices=['android', 'ios'],
type=str.lower,
help='Device type for testing',
dest='devicetype')
parseonlyparser.add_argument('--package-path',
help='Location of test application',
dest='packagepath')
parseonlyparser.add_argument('--package-name',
help='Classname of application',
dest='packagename')
parseonlyparser.add_argument('--exit-code',
help='Success exit code',
dest='expectedexitcode')
parseonlyparser.add_argument('--startup-iterations',
help='Startups to run (1+)',
type=int,
default=5,
dest='startupiterations')
self.add_common_arguments(parseonlyparser)
# inner loop command
innerloopparser = subparsers.add_parser(
const.INNERLOOP,
description=
'measure time to main and difference between two runs in a row')
self.add_common_arguments(innerloopparser)
# inner loop msbuild command
innerloopparser = subparsers.add_parser(
const.INNERLOOPMSBUILD,
description=
'measure time to main and difference between two runs in a row')
self.add_common_arguments(innerloopparser)
# dotnet watch command
dotnetwatchparser = subparsers.add_parser(
const.DOTNETWATCH,
description='measure time to main and time for hot reload')
self.add_common_arguments(dotnetwatchparser)
# sdk command
sdkparser = subparsers.add_parser(
const.SDK,
description='subcommands for sdk scenario',
formatter_class=RawTextHelpFormatter)
sdkparser.add_argument('sdktype',
choices=[
const.CLEAN_BUILD, const.BUILD_NO_CHANGE,
const.NEW_CONSOLE
],
type=str.lower,
help='''
clean_build: measure duration of building from source in each iteration
build_no_change: measure duration of building with existing output in each iteration
new_console: measure duration of creating a new console template
''')
self.add_common_arguments(sdkparser)
crossgenparser = subparsers.add_parser(
const.CROSSGEN,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen_arguments(crossgenparser)
self.add_common_arguments(crossgenparser)
crossgen2parser = subparsers.add_parser(
const.CROSSGEN2,
description='measure duration of the crossgen compilation',
formatter_class=RawTextHelpFormatter)
self.crossgen_arguments.add_crossgen2_arguments(crossgen2parser)
self.add_common_arguments(crossgen2parser)
sodparser = subparsers.add_parser(
const.SOD,
description=
'measure size on disk of the specified directory and its children')
sodparser.add_argument('--dirs',
dest='dirs',
type=str,
help=r'''
directories to measure separated by semicolon
ex: C:\repos\performance;C:\repos\runtime
''')
self.add_common_arguments(sodparser)
args = parser.parse_args()
if not args.testtype:
getLogger().error(
"Please specify a test type: %s. Type test.py <test type> -- help for more type-specific subcommands"
% testtypes)
sys.exit(1)
self.testtype = args.testtype
if self.testtype == const.SDK:
self.sdktype = args.sdktype
if self.testtype == const.CROSSGEN:
self.crossgen_arguments.parse_crossgen_args(args)
if self.testtype == const.CROSSGEN2:
self.crossgen_arguments.parse_crossgen2_args(args)
if self.testtype == const.SOD:
self.dirs = args.dirs
if self.testtype == const.DEVICESTARTUP:
self.packagepath = args.packagepath
self.packagename = args.packagename
self.devicetype = args.devicetype
self.expectedexitcode = args.expectedexitcode
self.startupiterations = args.startupiterations
if args.scenarioname:
self.scenarioname = args.scenarioname
def add_common_arguments(self, parser: ArgumentParser):
"Common arguments to add to subparsers"
parser.add_argument('--scenario-name', dest='scenarioname')
def run(self):
'''
Runs the specified scenario
'''
self.parseargs()
if self.testtype == const.INNERLOOP:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.INNERLOOPMSBUILD:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD],
apptorun='dotnet',
appargs='run --project %s' %
appfolder(self.traits.exename, self.traits.projext),
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
startup.runtests(self.traits)
if self.testtype == const.DOTNETWATCH:
startup = StartupWrapper()
self.traits.add_traits(
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH],
apptorun='dotnet',
appargs='watch -v',
innerloopcommand=pythoncommand(),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE))
self.traits.add_traits(workingdir=const.APPDIR)
startup.runtests(self.traits)
if self.testtype == const.STARTUP:
startup = StartupWrapper()
self.traits.add_traits(
overwrite=False,
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '',
scenarioname=self.scenarioname,
scenariotypename=const.SCENARIO_NAMES[const.STARTUP],
apptorun=publishedexe(self.traits.exename),
)
startup.runtests(self.traits)
elif self.testtype == const.SDK:
startup = StartupWrapper()
envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0'
envlistcleanbuild = ';'.join(
['MSBUILDDISABLENODEREUSE=1', envlistbuild])
# clean build
if self.sdktype == const.CLEAN_BUILD:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD),
apptorun=const.DOTNET,
appargs='build',
iterationsetup=pythoncommand(),
setupargs='%s %s setup_build' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR,
environmentvariables=envlistcleanbuild,
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# build(no changes)
if self.sdktype == const.BUILD_NO_CHANGE:
self.traits.add_traits(
overwrite=False,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE),
apptorun=const.DOTNET,
appargs='build',
workingdir=const.APPDIR,
environmentvariables=envlistbuild)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
# new console
if self.sdktype == const.NEW_CONSOLE:
self.traits.add_traits(
overwrite=False,
appargs='new console',
apptorun=const.DOTNET,
scenarioname=self.scenarioname,
scenariotypename='%s_%s' %
(const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE),
iterationsetup=pythoncommand(),
setupargs='%s %s setup_new' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
iterationcleanup=pythoncommand(),
cleanupargs='%s %s cleanup' %
('-3' if iswin() else '', const.ITERATION_SETUP_FILE),
workingdir=const.APPDIR)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN:
startup = StartupWrapper()
crossgenexe = 'crossgen%s' % extension()
crossgenargs = self.crossgen_arguments.get_crossgen_command_line()
coreroot = self.crossgen_arguments.coreroot
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
self.traits.add_traits(overwrite=True,
startupmetric=const.STARTUP_PROCESSTIME,
workingdir=coreroot,
appargs=' '.join(crossgenargs))
self.traits.add_traits(
overwrite=False,
scenarioname='Crossgen Throughput - %s' % scenario_filename,
scenariotypename='%s - %s' %
(const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename),
apptorun='%s\%s' % (coreroot, crossgenexe),
)
startup.runtests(self.traits)
elif self.testtype == const.CROSSGEN2:
startup = StartupWrapper()
scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename(
)
crossgen2args = self.crossgen_arguments.get_crossgen2_command_line(
)
compiletype = self.crossgen_arguments.crossgen2_compiletype()
scenarioname = 'Crossgen2 Throughput - %s - %s' % (
compiletype, scenario_filename)
if self.crossgen_arguments.singlethreaded:
scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (
compiletype, scenario_filename)
if compiletype == const.CROSSGEN2_COMPOSITE:
self.traits.add_traits(overwrite=True, skipprofile='true')
self.traits.add_traits(
overwrite=True,
startupmetric=const.STARTUP_CROSSGEN2,
workingdir=self.crossgen_arguments.coreroot,
appargs='%s %s' % (os.path.join(
'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args)))
self.traits.add_traits(
overwrite=False,
scenarioname=scenarioname,
apptorun=os.path.join(self.crossgen_arguments.coreroot,
'corerun%s' % extension()),
environmentvariables='COMPlus_EnableEventLog=1'
if not iswin() else '' # turn on clr user events
)
startup.runtests(self.traits)
elif self.testtype == const.DEVICESTARTUP:
getLogger().info("Clearing potential previous run nettraces")
for file in glob.glob(
os.path.join(const.TRACEDIR, 'PerfTest',
'trace*.nettrace')):
if exists(file):
getLogger().info("Removed: " +
os.path.join(const.TRACEDIR, file))
os.remove(file)
cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb']
adb = RunCommand(cmdline, verbose=True)
adb.run()
cmdline = [
adb.stdout.strip(), 'shell', 'mkdir', '-p', '/sdcard/PerfTest'
]
RunCommand(cmdline, verbose=True).run()
cmdline = xharnesscommand() + [
self.devicetype, 'install', '--app', self.packagepath,
'--package-name', self.packagename, '-o', const.TRACEDIR, '-v'
]
RunCommand(cmdline, verbose=True).run()
for i in range(self.startupiterations):
cmdline = xharnesscommand() + [
'android', 'run', '-o', const.TRACEDIR, '--package-name',
self.packagename, '-v',
'--arg=env:COMPlus_EnableEventPipe=1',
'--arg=env:COMPlus_EventPipeOutputStreaming=1',
'--arg=env:COMPlus_EventPipeOutputPath=/sdcard/PerfTest/trace%s.nettrace'
% (i + 1), '--arg=env:COMPlus_EventPipeCircularMB=10',
'--arg=env:COMPlus_EventPipeConfig=Microsoft-Windows-DotNETRuntime:10:5',
'--expected-exit-code', self.expectedexitcode, '--dev-out',
'/sdcard/PerfTest/'
]
RunCommand(cmdline, verbose=True).run()
cmdline = xharnesscommand() + [
'android', 'uninstall', '--package-name', self.packagename
]
RunCommand(cmdline, verbose=True).run()
cmdline = [
adb.stdout.strip(), 'shell', 'rm', '-r', '/sdcard/PerfTest'
]
RunCommand(cmdline, verbose=True).run()
startup = StartupWrapper()
self.traits.add_traits(
overwrite=True,
apptorun="app",
startupmetric=const.STARTUP_DEVICETIMETOMAIN,
tracefolder='PerfTest/',
tracename='trace*.nettrace',
scenarioname='Device Startup - Android %s' %
(self.packagename))
startup.parsetraces(self.traits)
elif self.testtype == const.SOD:
sod = SODWrapper()
builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None
if not builtdir:
builtdir = const.BINDIR if os.path.exists(
const.BINDIR) else None
if not (self.dirs or builtdir):
raise Exception(
"Dirs was not passed in and neither %s nor %s exist" %
(const.PUBDIR, const.BINDIR))
sod.runtests(scenarioname=self.scenarioname,
dirs=self.dirs or builtdir,
artifact=self.traits.artifact)
