def shutdown_dotnet_servers(): # shutdown_server(verbose=True) # This is the correct way to shut down dotnet build servers, but it has been disabled due to https://github.com/dotnet/sdk/issues/10573 getLogger().info("Shutting down dotnet build servers...") if iswin(): os.system('TASKKILL /F /T /IM dotnet.exe || TASKKILL /F /T /IM VSTest.Console.exe || TASKKILL /F /T /IM msbuild.exe') else: os.system('killall -9 dotnet || killall -9 VSTest.Console || killall -9 msbuild')
def getruntimeidentifier(): rid = None if iswin(): rid = 'win-' elif sys.platform == 'linux' or sys.platform == 'linux2': rid = 'linux-' else: raise Exception('Platform %s not supported.' % sys.platform) if 'aarch64' in platform.machine() or os.environ.get('PERFLAB_BUILDARCH') == 'arm64': rid += 'arm64' elif platform.machine().endswith('64'): rid += 'x64' elif platform.machine().endswith('86'): rid += 'x86' else: raise Exception('Machine %s not supported.' % platform.machine()) return rid
def pythoncommand(): if iswin(): return 'py' else: return 'python3'
def run(self): ''' Runs the specified scenario ''' self.parseargs() if self.testtype == const.INNERLOOP: startup = StartupWrapper() self.traits.add_traits(scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.INNERLOOPMSBUILD: startup = StartupWrapper() self.traits.add_traits(scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.DOTNETWATCH: startup = StartupWrapper() self.traits.add_traits(scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH], apptorun='dotnet', appargs='watch -v', innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) self.traits.add_traits(workingdir = const.APPDIR) startup.runtests(self.traits) if self.testtype == const.STARTUP: startup = StartupWrapper() self.traits.add_traits(overwrite=False, environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '', scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.STARTUP], apptorun=publishedexe(self.traits.exename), ) startup.runtests(self.traits) elif self.testtype == const.SDK: startup = StartupWrapper() envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0' envlistcleanbuild = ';'.join(['MSBUILDDISABLENODEREUSE=1', envlistbuild]) # clean build if self.sdktype == const.CLEAN_BUILD: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD), apptorun=const.DOTNET, appargs='build', iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR, environmentvariables=envlistcleanbuild, ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # build(no changes) if self.sdktype == const.BUILD_NO_CHANGE: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE), apptorun=const.DOTNET, appargs='build', workingdir=const.APPDIR, environmentvariables=envlistbuild ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # new console if self.sdktype == const.NEW_CONSOLE: self.traits.add_traits( overwrite=False, appargs='new console', apptorun=const.DOTNET, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE), iterationsetup=pythoncommand(), setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN: startup = StartupWrapper() crossgenexe = 'crossgen%s' % extension() crossgenargs = self.crossgen_arguments.get_crossgen_command_line() coreroot = self.crossgen_arguments.coreroot scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename() self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME, workingdir=coreroot, appargs=' '.join(crossgenargs) ) self.traits.add_traits(overwrite=False, scenarioname='Crossgen Throughput - %s' % scenario_filename, scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename), apptorun='%s\%s' % (coreroot, crossgenexe), ) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN2: startup = StartupWrapper() scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename() crossgen2args = self.crossgen_arguments.get_crossgen2_command_line() compiletype = self.crossgen_arguments.crossgen2_compiletype() scenarioname = 'Crossgen2 Throughput - %s - %s' % (compiletype, scenario_filename) if self.crossgen_arguments.singlethreaded: scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % (compiletype, scenario_filename) if compiletype == const.CROSSGEN2_COMPOSITE: self.traits.add_traits(overwrite=True, skipprofile='true') self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_CROSSGEN2, workingdir=self.crossgen_arguments.coreroot, appargs='%s %s' % (os.path.join('crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args)) ) self.traits.add_traits(overwrite=False, scenarioname=scenarioname, apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()), environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events ) startup.runtests(self.traits) elif self.testtype == const.DEVICESTARTUP and self.devicetype == 'android': # ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc # Regex used to split the response from starting the activity and saving each value #Example: # Starting: Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity } # Status: ok # LaunchState: COLD # Activity: net.dot.HelloAndroid/net.dot.MainActivity # TotalTime: 241 # WaitTime: 242 # Complete # Saves: [Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }, ok, COLD, net.dot.HelloAndroid/net.dot.MainActivity, 241, 242] # Split results (start at 0) (List is Starting (Intent activity), Status (ok...), LaunchState ([HOT, COLD, WARM]), Activity (started activity name), TotalTime(toFrameOne), WaitTime(toFullLoad)) runSplitRegex = ":\s(.+)" screenWasOff = False getLogger().info("Clearing potential previous run nettraces") for file in glob.glob(os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')): if exists(file): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) cmdline = xharnesscommand() + ['android', 'state', '--adb'] adb = RunCommand(cmdline, verbose=True) adb.run() # Do not remove, XHarness install seems to fail without an adb command called before the xharness command getLogger().info("Preparing ADB") cmdline = [ adb.stdout.strip(), 'shell', 'wm', 'size' ] RunCommand(cmdline, verbose=True).run() # Get animation values getLogger().info("Getting Values we will need set specifically") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] window_animation_scale_cmd = RunCommand(cmdline, verbose=True) window_animation_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] transition_animation_scale_cmd = RunCommand(cmdline, verbose=True) transition_animation_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] animator_duration_scale_cmd = RunCommand(cmdline, verbose=True) animator_duration_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'system', 'screen_off_timeout' ] screen_off_timeout_cmd = RunCommand(cmdline, verbose=True) screen_off_timeout_cmd.run() getLogger().info(f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}, screen timeout {screen_off_timeout_cmd.stdout.strip()}") # Make sure animations are set to 1 or disabled getLogger().info("Setting needed values") if(self.animationsdisabled): animationValue = 0 else: animationValue = 1 minimumTimeoutValue = 2 * 60 * 1000 # milliseconds cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'transition_animation_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'animator_duration_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'system', 'screen_off_timeout', str(minimumTimeoutValue) ] if(minimumTimeoutValue > int(screen_off_timeout_cmd.stdout.strip())): getLogger().info("Screen off value is lower than minimum time, setting to higher time") RunCommand(cmdline, verbose=True).run() # Check for success getLogger().info("Getting animation values to verify it worked") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] windowSetValue = RunCommand(cmdline, verbose=True) windowSetValue.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] transitionSetValue = RunCommand(cmdline, verbose=True) transitionSetValue.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] animatorSetValue = RunCommand(cmdline, verbose=True) animatorSetValue.run() if(int(windowSetValue.stdout.strip()) != animationValue or int(transitionSetValue.stdout.strip()) != animationValue or int(animatorSetValue.stdout.strip()) != animationValue): # Setting the values didn't work, error out getLogger().exception(f"Failed to set animation values to {animationValue}.") raise Exception(f"Failed to set animation values to {animationValue}.") else: getLogger().info(f"Animation values successfully set to {animationValue}.") try: installCmd = xharnesscommand() + [ 'android', 'install', '--app', self.packagepath, '--package-name', self.packagename, '-o', const.TRACEDIR, '-v' ] RunCommand(installCmd, verbose=True).run() getLogger().info("Completed install, running shell.") cmdline = [ adb.stdout.strip(), 'shell', f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' ] getActivity = RunCommand(cmdline, verbose=True) getActivity.run() getLogger().info(f"Target Activity {getActivity.stdout}") # More setup stuff checkScreenOnCmd = [ adb.stdout.strip(), 'shell', f'dumpsys input_method | grep mInteractive' ] checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() keyInputCmd = [ adb.stdout.strip(), 'shell', 'input', 'keyevent' ] if("mInteractive=false" in checkScreenOn.stdout): # Turn on the screen to make interactive and see if it worked getLogger().info("Screen was off, turning on.") screenWasOff = True RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock the screen with menu key (only works if it is not a password lock) checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() if("mInteractive=false" in checkScreenOn.stdout): getLogger().exception("Failed to make screen interactive.") raise Exception("Failed to make screen interactive.") # Actual testing some run stuff getLogger().info("Test run to check if permissions are needed") activityname = getActivity.stdout # -W in the start command waits for the app to finish initial draw. startAppCmd = [ adb.stdout.strip(), 'shell', 'am', 'start-activity', '-W', '-n', activityname ] testRun = RunCommand(startAppCmd, verbose=True) testRun.run() testRunStats = re.findall(runSplitRegex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle stopAppCmd = [ adb.stdout.strip(), 'shell', 'am', 'force-stop', self.packagename ] RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen time.sleep(1) RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen time.sleep(1) # Check to make sure it worked testRun = RunCommand(startAppCmd, verbose=True) testRun.run() testRunStats = re.findall(runSplitRegex, testRun.stdout) getLogger().info(f"Test run activity: {testRunStats[3]}") RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") # Create the fullydrawn command fullyDrawnRetrieveCmd = [ adb.stdout.strip(), 'shell', f"logcat -d | grep 'ActivityTaskManager: Fully drawn {self.packagename}'" ] basicStartupRetrieveCmd = [ adb.stdout.strip(), 'shell', f"logcat -d | grep 'ActivityTaskManager: Displayed {activityname}'" ] clearLogsCmd = [ adb.stdout.strip(), 'logcat', '-c' ] allResults = [] for i in range(self.startupiterations): # Clear logs RunCommand(clearLogsCmd, verbose=True).run() startStats = RunCommand(startAppCmd, verbose=True) startStats.run() # Make sure we cold started (TODO Add other starts) if("LaunchState: COLD" not in startStats.stdout): getLogger().error("App Start not COLD!") # Save the results and get them from the log if self.usefullydrawntime: time.sleep(self.fullyDrawnDelaySecMax) # Start command doesn't wait for fully drawn report, force a wait for it. -W in the start command waits for the app to finish initial draw. RunCommand(stopAppCmd, verbose=True).run() if self.usefullydrawntime: retrieveTimeCmd = RunCommand(fullyDrawnRetrieveCmd, verbose=True) else: retrieveTimeCmd = RunCommand(basicStartupRetrieveCmd, verbose=True) retrieveTimeCmd.run() dirtyCapture = re.search("\+(\d*s?\d+)ms", retrieveTimeCmd.stdout) if not dirtyCapture: raise Exception("Failed to capture the reported start time!") formattedTime = f"TotalTime: {dirtyCapture.group(1).replace('s', '')}\n" allResults.append(formattedTime) # append TotalTime: (TIME) time.sleep(3) # Delay in seconds for ensuring a cold start finally: getLogger().info("Stopping App for uninstall") RunCommand(stopAppCmd, verbose=True).run() getLogger().info("Uninstalling app") uninstallAppCmd = xharnesscommand() + [ 'android', 'uninstall', '--package-name', self.packagename ] RunCommand(uninstallAppCmd, verbose=True).run() # Reset animation values getLogger().info("Resetting animation values to pretest values") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'window_animation_scale', window_animation_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'transition_animation_scale', transition_animation_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'animator_duration_scale', animator_duration_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'system', 'screen_off_timeout', screen_off_timeout_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() if screenWasOff: RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off # Create traces to store the data so we can keep the current general parse trace flow getLogger().info(f"Logs: \n{allResults}") os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True) traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w") for result in allResults: traceFile.write(result) traceFile.close() startup = StartupWrapper() self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) startup.parsetraces(self.traits) elif self.testtype == const.DEVICESTARTUP and self.devicetype == 'ios': getLogger().info("Clearing potential previous run nettraces") for file in glob.glob(os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')): if exists(file): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) if not exists(const.TMPDIR): os.mkdir(const.TMPDIR) getLogger().info("Clearing potential previous run *.logarchive") for logarchive in glob.glob(os.path.join(const.TMPDIR, '*.logarchive')): if exists(logarchive): getLogger().info("Removed: " + os.path.join(const.TMPDIR, logarchive)) rmtree(logarchive) getLogger().info("Checking device state.") cmdline = xharnesscommand() + ['apple', 'state'] adb = RunCommand(cmdline, verbose=True) adb.run() getLogger().info("Installing app on device.") installCmd = xharnesscommand() + [ 'apple', 'install', '--app', self.packagepath, '--target', 'ios-device', '-o', const.TRACEDIR, '-v' ] RunCommand(installCmd, verbose=True).run() getLogger().info("Completed install.") allResults = [] for i in range(self.startupiterations + 1): # adding one iteration to account for the warmup iteration getLogger().info("Waiting 10 secs to ensure we're not getting confused with previous app run.") time.sleep(10) getLogger().info(f"Collect startup data for iteration {i}.") runCmdTimestamp = datetime.now() runCmd = xharnesscommand() + [ 'apple', 'mlaunch', '--', f'--launchdevbundleid={self.packagename}', ] runCmdCommand = RunCommand(runCmd, verbose=True) runCmdCommand.run() app_pid_search = re.search("Launched application.*with pid (?P<app_pid>\d+)", runCmdCommand.stdout) app_pid = int(app_pid_search.group('app_pid')) logarchive_filename = os.path.join(const.TMPDIR, f'iteration{i}.logarchive') getLogger().info(f"Waiting 5 secs to ensure app with PID {app_pid} is fully started.") time.sleep(5) collectCmd = [ 'sudo', 'log', 'collect', '--device', '--start', runCmdTimestamp.strftime("%Y-%m-%d %H:%M:%S"), '--output', logarchive_filename, ] RunCommand(collectCmd, verbose=True).run() getLogger().info(f"Kill app with PID {app_pid}.") killCmd = xharnesscommand() + [ 'apple', 'mlaunch', '--', f'--killdev={app_pid}', ] killCmdCommand = RunCommand(killCmd, verbose=True) killCmdCommand.run() # Process Data # There are four watchdog events from SpringBoard during an application startup: # # [application<net.dot.maui>:770] [realTime] Now monitoring resource allowance of 20.00s (at refreshInterval -1.00s) # [application<net.dot.maui>:770] [realTime] Stopped monitoring. # [application<net.dot.maui>:770] [realTime] Now monitoring resource allowance of 19.28s (at refreshInterval -1.00s) # [application<net.dot.maui>:770] [realTime] Stopped monitoring. # # The first two are monitoring the time it takes the OS to create the process, load .dylibs and call into the app's main() # The second two are monitoring the time it takes the app to draw the first frame of UI from main() # # An app has 20 seconds to complete this sequence or the OS will kill the app. # We collect these log events to do our measurements. logShowCmd = [ 'log', 'show', '--predicate', '(process == "SpringBoard") && (category == "Watchdog")', '--info', '--style', 'ndjson', logarchive_filename, ] logShowCmdCommand = RunCommand(logShowCmd, verbose=True) logShowCmdCommand.run() events = [] for line in logShowCmdCommand.stdout.splitlines(): try: lineData = json.loads(line) if 'Now monitoring resource allowance' in lineData['eventMessage'] or 'Stopped monitoring' in lineData['eventMessage']: events.append (lineData) except: break # the startup measurement relies on the date/time of the device to be pretty much in sync with the host # since we use the timestamps from the host to decide which parts of the device log to get and # we then use that to calculate the time delta from watchdog events if len(events) != 4: raise Exception("Didn't get the right amount of watchdog events, this could mean the app crashed or the device clock is not in sync with the host.") timeToMainEventStart = events[0] timeToMainEventStop = events[1] timeToFirstDrawEventStart = events[2] timeToFirstDrawEventStop = events[3] # validate log messages if f'application<{self.packagename}>:{app_pid}' not in timeToMainEventStart['eventMessage'] or 'Now monitoring resource allowance of 20.00s' not in timeToMainEventStart['eventMessage']: raise Exception(f"Invalid timeToMainEventStart: {timeToMainEventStart['eventMessage']}") if f'application<{self.packagename}>:{app_pid}' not in timeToMainEventStop['eventMessage'] or 'Stopped monitoring' not in timeToMainEventStop['eventMessage']: raise Exception(f"Invalid timeToMainEventStop: {timeToMainEventStop['eventMessage']}") if f'application<{self.packagename}>:{app_pid}' not in timeToFirstDrawEventStart['eventMessage'] or 'Now monitoring resource allowance of' not in timeToFirstDrawEventStart['eventMessage']: raise Exception(f"Invalid timeToFirstDrawEventStart: {timeToFirstDrawEventStart['eventMessage']}") if f'application<{self.packagename}>:{app_pid}' not in timeToFirstDrawEventStop['eventMessage'] or 'Stopped monitoring' not in timeToFirstDrawEventStop['eventMessage']: raise Exception(f"Invalid timeToFirstDrawEventStop: {timeToFirstDrawEventStop['eventMessage']}") timeToMainEventStartDateTime = datetime.strptime(timeToMainEventStart['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') timeToMainEventEndDateTime = datetime.strptime(timeToMainEventStop['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') timeToMainMilliseconds = (timeToMainEventEndDateTime - timeToMainEventStartDateTime).total_seconds() * 1000 timeToFirstDrawEventStartDateTime = datetime.strptime(timeToFirstDrawEventStart['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') timeToFirstDrawEventEndDateTime = datetime.strptime(timeToFirstDrawEventStop['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') timeToFirstDrawMilliseconds = (timeToFirstDrawEventEndDateTime - timeToFirstDrawEventStartDateTime).total_seconds() * 1000 if self.usefullydrawntime: # grab log event with the magic string in it logShowMagicStringCmd = [ 'log', 'show', '--predicate', f'(processIdentifier == {app_pid}) && (composedMessage contains "{self.fullyDrawnMagicString}")', '--info', '--style', 'ndjson', logarchive_filename, ] logShowMagicStringCmd = RunCommand(logShowMagicStringCmd, verbose=True) logShowMagicStringCmd.run() magicStringEvent = '' for line in logShowMagicStringCmd.stdout.splitlines(): try: lineData = json.loads(line) if self.fullyDrawnMagicString in lineData['eventMessage']: magicStringEvent = lineData except: break if magicStringEvent == '': raise Exception("Didn't get the fully-drawn magic string event.") timeToMagicStringEventDateTime = datetime.strptime(magicStringEvent['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') # startup time is time to the magic string event totalTimeMilliseconds = (timeToMagicStringEventDateTime - timeToMainEventStartDateTime).total_seconds() * 1000 else: # startup time is time to first draw totalTimeMilliseconds = timeToMainMilliseconds + timeToFirstDrawMilliseconds if i == 0: # ignore the warmup iteration getLogger().info(f'Warmup iteration took {totalTimeMilliseconds}') else: # TODO: this isn't really a COLD run, we should have separate measurements for starting the app right after install launchState = 'COLD' allResults.append(f'LaunchState: {launchState}\nTotalTime: {int(totalTimeMilliseconds)}\nTimeToMain: {int(timeToMainMilliseconds)}\n\n') # Done with testing, uninstall the app getLogger().info("Uninstalling app") uninstallAppCmd = xharnesscommand() + [ 'apple', 'uninstall', '--app', self.packagename, '--target', 'ios-device', '-o', const.TRACEDIR, '-v' ] RunCommand(uninstallAppCmd, verbose=True).run() # Create traces to store the data so we can keep the current general parse trace flow getLogger().info(f"Logs: \n{allResults}") os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True) traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w") for result in allResults: traceFile.write(result) traceFile.close() startup = StartupWrapper() self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) startup.parsetraces(self.traits) elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None if not builtdir: builtdir = const.BINDIR if os.path.exists(const.BINDIR) else None if not (self.dirs or builtdir): raise Exception("Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR)) sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)
def run(self): ''' Runs the specified scenario ''' self.parseargs() if self.testtype == const.INNERLOOP: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.INNERLOOPMSBUILD: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.DOTNETWATCH: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH], apptorun='dotnet', appargs='watch -v', innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) self.traits.add_traits(workingdir=const.APPDIR) startup.runtests(self.traits) if self.testtype == const.STARTUP: startup = StartupWrapper() self.traits.add_traits( overwrite=False, environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '', scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.STARTUP], apptorun=publishedexe(self.traits.exename), ) startup.runtests(self.traits) elif self.testtype == const.SDK: startup = StartupWrapper() envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0' envlistcleanbuild = ';'.join( ['MSBUILDDISABLENODEREUSE=1', envlistbuild]) # clean build if self.sdktype == const.CLEAN_BUILD: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD), apptorun=const.DOTNET, appargs='build', iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR, environmentvariables=envlistcleanbuild, ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # build(no changes) if self.sdktype == const.BUILD_NO_CHANGE: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE), apptorun=const.DOTNET, appargs='build', workingdir=const.APPDIR, environmentvariables=envlistbuild) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # new console if self.sdktype == const.NEW_CONSOLE: self.traits.add_traits( overwrite=False, appargs='new console', apptorun=const.DOTNET, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE), iterationsetup=pythoncommand(), setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN: startup = StartupWrapper() crossgenexe = 'crossgen%s' % extension() crossgenargs = self.crossgen_arguments.get_crossgen_command_line() coreroot = self.crossgen_arguments.coreroot scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME, workingdir=coreroot, appargs=' '.join(crossgenargs)) self.traits.add_traits( overwrite=False, scenarioname='Crossgen Throughput - %s' % scenario_filename, scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename), apptorun='%s\%s' % (coreroot, crossgenexe), ) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN2: startup = StartupWrapper() scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) crossgen2args = self.crossgen_arguments.get_crossgen2_command_line( ) compiletype = self.crossgen_arguments.crossgen2_compiletype() scenarioname = 'Crossgen2 Throughput - %s - %s' % ( compiletype, scenario_filename) if self.crossgen_arguments.singlethreaded: scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % ( compiletype, scenario_filename) if compiletype == const.CROSSGEN2_COMPOSITE: self.traits.add_traits(overwrite=True, skipprofile='true') self.traits.add_traits( overwrite=True, startupmetric=const.STARTUP_CROSSGEN2, workingdir=self.crossgen_arguments.coreroot, appargs='%s %s' % (os.path.join( 'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args))) self.traits.add_traits( overwrite=False, scenarioname=scenarioname, apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()), environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events ) startup.runtests(self.traits) elif self.testtype == const.DEVICESTARTUP: # ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc # Regex used to split the response from starting the activity and saving each value #Example: # Starting: Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity } # Status: ok # LaunchState: COLD # Activity: net.dot.HelloAndroid/net.dot.MainActivity # TotalTime: 241 # WaitTime: 242 # Complete # Saves: [Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }, ok, COLD, net.dot.HelloAndroid/net.dot.MainActivity, 241, 242] # Split results (start at 0) (List is Starting (Intent activity), Status (ok...), LaunchState ([HOT, COLD, WARM]), Activity (started activity name), TotalTime(toFrameOne), WaitTime(toFullLoad)) runSplitRegex = ":\s(.+)" screenWasOff = False getLogger().info("Clearing potential previous run nettraces") for file in glob.glob( os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')): if exists(file): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb'] adb = RunCommand(cmdline, verbose=True) adb.run() # Do not remove, XHarness install seems to fail without an adb command called before the xharness command getLogger().info("Preparing ADB") cmdline = [adb.stdout.strip(), 'shell', 'wm', 'size'] RunCommand(cmdline, verbose=True).run() # Get animation values getLogger().info("Getting animation values") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] window_animation_scale_cmd = RunCommand(cmdline, verbose=True) window_animation_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] transition_animation_scale_cmd = RunCommand(cmdline, verbose=True) transition_animation_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] animator_duration_scale_cmd = RunCommand(cmdline, verbose=True) animator_duration_scale_cmd.run() getLogger().info( f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}" ) # Make sure animations are set to 1 or disabled getLogger().info("Setting animation values") if (self.animationsdisabled): animationValue = 0 else: animationValue = 1 cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'transition_animation_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'animator_duration_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() # Check for success getLogger().info("Getting animation values to verify it worked") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] windowSetValue = RunCommand(cmdline, verbose=True) windowSetValue.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] transitionSetValue = RunCommand(cmdline, verbose=True) transitionSetValue.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] animatorSetValue = RunCommand(cmdline, verbose=True) animatorSetValue.run() if (int(windowSetValue.stdout.strip()) != animationValue or int(transitionSetValue.stdout.strip()) != animationValue or int(animatorSetValue.stdout.strip()) != animationValue): # Setting the values didn't work, error out getLogger().exception( f"Failed to set animation values to {animationValue}.") exit(-1) else: getLogger().info( f"Animation values successfully set to {animationValue}.") installCmd = xharnesscommand() + [ self.devicetype, 'install', '--app', self.packagepath, '--package-name', self.packagename, '-o', const.TRACEDIR, '-v' ] RunCommand(installCmd, verbose=True).run() getLogger().info("Completed install, running shell.") cmdline = [ adb.stdout.strip(), 'shell', f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' ] getActivity = RunCommand(cmdline, verbose=True) getActivity.run() getLogger().info(f"Target Activity {getActivity.stdout}") # More setup stuff checkScreenOnCmd = [ adb.stdout.strip(), 'shell', f'dumpsys input_method | grep mInteractive' ] checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() keyInputCmd = [adb.stdout.strip(), 'shell', 'input', 'keyevent'] if ("mInteractive=false" in checkScreenOn.stdout): # Turn on the screen to make interactive and see if it worked getLogger().info("Screen was off, turning on.") screenWasOff = True RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key RunCommand(keyInputCmd + ['82'], verbose=True).run( ) # Unlock the screen with menu key (only works if it is not a password lock) checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() if ("mInteractive=false" in checkScreenOn.stdout): getLogger().exception("Failed to make screen interactive.") exit(-1) # Actual testing some run stuff getLogger().info("Test run to check if permissions are needed") activityname = getActivity.stdout startAppCmd = [ adb.stdout.strip(), 'shell', 'am', 'start-activity', '-W', '-n', activityname ] testRun = RunCommand(startAppCmd, verbose=True) testRun.run() testRunStats = re.findall( runSplitRegex, testRun.stdout ) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") stopAppCmd = [ adb.stdout.strip(), 'shell', 'am', 'force-stop', self.packagename ] RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['66'], verbose=True).run( ) # Press enter to close main perm screen time.sleep(1) RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['66'], verbose=True).run( ) # Press enter to close out of second screen time.sleep(1) # Check to make sure it worked testRun = RunCommand(startAppCmd, verbose=True) testRun.run() testRunStats = re.findall(runSplitRegex, testRun.stdout) getLogger().info(f"Test run activity: {testRunStats[3]}") RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[ 3]: getLogger().exception( "Failed to get past permission screen, run locally to see if enough next button presses were used." ) exit(-1) allResults = [] for i in range(self.startupiterations): startStats = RunCommand(startAppCmd, verbose=True) startStats.run() RunCommand(stopAppCmd, verbose=True).run() allResults.append( startStats.stdout ) # Save results (List is Intent, Status, LaunchState Activity, TotalTime, WaitTime) time.sleep(3) # Delay in seconds for ensuring a cold start getLogger().info("Stopping App for uninstall") RunCommand(stopAppCmd, verbose=True).run() getLogger().info("Uninstalling app") uninstallAppCmd = xharnesscommand() + [ 'android', 'uninstall', '--package-name', self.packagename ] RunCommand(uninstallAppCmd, verbose=True).run() # Reset animation values getLogger().info("Resetting animation values to pretest values") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'window_animation_scale', window_animation_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'transition_animation_scale', transition_animation_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'animator_duration_scale', animator_duration_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() if screenWasOff: RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off # Create traces to store the data so we can keep the current general parse trace flow getLogger().info(f"Logs: \n{allResults}") os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True) traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w") for result in allResults: traceFile.write(result) traceFile.close() startup = StartupWrapper() self.traits.add_traits( overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) startup.parsetraces(self.traits) elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None if not builtdir: builtdir = const.BINDIR if os.path.exists( const.BINDIR) else None if not (self.dirs or builtdir): raise Exception( "Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR)) sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)
def run(self): ''' Runs the specified scenario ''' self.parseargs() if self.testtype == const.INNERLOOP: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.INNERLOOPMSBUILD: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.DOTNETWATCH: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH], apptorun='dotnet', appargs='watch -v', innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) self.traits.add_traits(workingdir=const.APPDIR) startup.runtests(self.traits) if self.testtype == const.STARTUP: startup = StartupWrapper() self.traits.add_traits( overwrite=False, environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '', scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.STARTUP], apptorun=publishedexe(self.traits.exename), ) startup.runtests(self.traits) elif self.testtype == const.SDK: startup = StartupWrapper() envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0' envlistcleanbuild = ';'.join( ['MSBUILDDISABLENODEREUSE=1', envlistbuild]) # clean build if self.sdktype == const.CLEAN_BUILD: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD), apptorun=const.DOTNET, appargs='build', iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR, environmentvariables=envlistcleanbuild, ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # build(no changes) if self.sdktype == const.BUILD_NO_CHANGE: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE), apptorun=const.DOTNET, appargs='build', workingdir=const.APPDIR, environmentvariables=envlistbuild) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # new console if self.sdktype == const.NEW_CONSOLE: self.traits.add_traits( overwrite=False, appargs='new console', apptorun=const.DOTNET, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE), iterationsetup=pythoncommand(), setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN: startup = StartupWrapper() crossgenexe = 'crossgen%s' % extension() crossgenargs = self.crossgen_arguments.get_crossgen_command_line() coreroot = self.crossgen_arguments.coreroot scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME, workingdir=coreroot, appargs=' '.join(crossgenargs)) self.traits.add_traits( overwrite=False, scenarioname='Crossgen Throughput - %s' % scenario_filename, scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename), apptorun='%s\%s' % (coreroot, crossgenexe), ) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN2: startup = StartupWrapper() scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) crossgen2args = self.crossgen_arguments.get_crossgen2_command_line( ) compiletype = self.crossgen_arguments.crossgen2_compiletype() scenarioname = 'Crossgen2 Throughput - %s - %s' % ( compiletype, scenario_filename) if self.crossgen_arguments.singlethreaded: scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % ( compiletype, scenario_filename) if compiletype == const.CROSSGEN2_COMPOSITE: self.traits.add_traits(overwrite=True, skipprofile='true') self.traits.add_traits( overwrite=True, startupmetric=const.STARTUP_CROSSGEN2, workingdir=self.crossgen_arguments.coreroot, appargs='%s %s' % (os.path.join( 'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args))) self.traits.add_traits( overwrite=False, scenarioname=scenarioname, apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()), environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events ) startup.runtests(self.traits) elif self.testtype == const.DEVICESTARTUP: getLogger().info("Clearing potential previous run nettraces") for file in glob.glob( os.path.join(const.TRACEDIR, 'PerfTest', 'trace*.nettrace')): if exists(file): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb'] adb = RunCommand(cmdline, verbose=True) adb.run() cmdline = [ adb.stdout.strip(), 'shell', 'mkdir', '-p', '/sdcard/PerfTest' ] RunCommand(cmdline, verbose=True).run() cmdline = xharnesscommand() + [ self.devicetype, 'install', '--app', self.packagepath, '--package-name', self.packagename, '-o', const.TRACEDIR, '-v' ] RunCommand(cmdline, verbose=True).run() for i in range(self.startupiterations): cmdline = xharnesscommand() + [ 'android', 'run', '-o', const.TRACEDIR, '--package-name', self.packagename, '-v', '--arg=env:COMPlus_EnableEventPipe=1', '--arg=env:COMPlus_EventPipeOutputStreaming=1', '--arg=env:COMPlus_EventPipeOutputPath=/sdcard/PerfTest/trace%s.nettrace' % (i + 1), '--arg=env:COMPlus_EventPipeCircularMB=10', '--arg=env:COMPlus_EventPipeConfig=Microsoft-Windows-DotNETRuntime:10:5', '--expected-exit-code', self.expectedexitcode, '--dev-out', '/sdcard/PerfTest/' ] RunCommand(cmdline, verbose=True).run() cmdline = xharnesscommand() + [ 'android', 'uninstall', '--package-name', self.packagename ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'rm', '-r', '/sdcard/PerfTest' ] RunCommand(cmdline, verbose=True).run() startup = StartupWrapper() self.traits.add_traits( overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='trace*.nettrace', scenarioname='Device Startup - Android %s' % (self.packagename)) startup.parsetraces(self.traits) elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None if not builtdir: builtdir = const.BINDIR if os.path.exists( const.BINDIR) else None if not (self.dirs or builtdir): raise Exception( "Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR)) sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)
def run(self): ''' Runs the specified scenario ''' self.parseargs() if self.testtype == const.INNERLOOP: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP], apptorun='dotnet', appargs='run -p %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.INNERLOOPMSBUILD: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD], apptorun='dotnet', appargs='run -p %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.DOTNETWATCH: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH], apptorun='dotnet', appargs='watch', innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) self.traits.add_traits(workingdir=const.APPDIR) startup.runtests(self.traits) if self.testtype == const.STARTUP: startup = StartupWrapper() self.traits.add_traits( overwrite=False, environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '', scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.STARTUP], apptorun=publishedexe(self.traits.exename), ) startup.runtests(self.traits) elif self.testtype == const.SDK: startup = StartupWrapper() envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0' envlistcleanbuild = ';'.join( ['MSBUILDDISABLENODEREUSE=1', envlistbuild]) # clean build if self.sdktype == const.CLEAN_BUILD: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD), apptorun=const.DOTNET, appargs='build', iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR, environmentvariables=envlistcleanbuild, ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # build(no changes) if self.sdktype == const.BUILD_NO_CHANGE: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE), apptorun=const.DOTNET, appargs='build', workingdir=const.APPDIR, environmentvariables=envlistbuild) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # new console if self.sdktype == const.NEW_CONSOLE: self.traits.add_traits( overwrite=False, appargs='new console', apptorun=const.DOTNET, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE), iterationsetup=pythoncommand(), setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN: startup = StartupWrapper() crossgenexe = 'crossgen%s' % extension() crossgenargs = self.crossgen_arguments.get_crossgen_command_line() coreroot = self.crossgen_arguments.coreroot scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME, workingdir=coreroot, appargs=' '.join(crossgenargs)) self.traits.add_traits( overwrite=False, scenarioname='Crossgen Throughput - %s' % scenario_filename, scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename), apptorun='%s\%s' % (coreroot, crossgenexe), ) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN2: startup = StartupWrapper() scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) crossgen2args = self.crossgen_arguments.get_crossgen2_command_line( ) compiletype = self.crossgen_arguments.crossgen2_compiletype() scenarioname = 'Crossgen2 Throughput - %s - %s' % ( compiletype, scenario_filename) if self.crossgen_arguments.singlethreaded: scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % ( compiletype, scenario_filename) if compiletype == const.CROSSGEN2_COMPOSITE: self.traits.add_traits(overwrite=True, skipprofile='true') self.traits.add_traits( overwrite=True, startupmetric=const.STARTUP_CROSSGEN2, workingdir=self.crossgen_arguments.coreroot, appargs='%s %s' % (os.path.join( 'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args))) self.traits.add_traits( overwrite=False, scenarioname=scenarioname, apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()), environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events ) startup.runtests(self.traits) elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None if not builtdir: builtdir = const.BINDIR if os.path.exists( const.BINDIR) else None if not (self.dirs or builtdir): raise Exception( "Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR)) sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)