def install( architecture: str, channels: list, versions: str, verbose: bool, install_dir: str = None) -> None: ''' Downloads dotnet cli into the tools folder. ''' __log_script_header("Downloading DotNet Cli") if not install_dir: install_dir = __get_directory(architecture) if not path.exists(install_dir): makedirs(install_dir) getLogger().info("DotNet Install Path: '%s'", install_dir) # Download appropriate dotnet install script dotnetInstallScriptExtension = '.ps1' if platform == 'win32' else '.sh' dotnetInstallScriptName = 'dotnet-install' + dotnetInstallScriptExtension url = 'https://dot.net/v1/' dotnetInstallScriptUrl = url + dotnetInstallScriptName dotnetInstallScriptPath = path.join(install_dir, dotnetInstallScriptName) getLogger().info('Downloading %s', dotnetInstallScriptUrl) count = 0 while count < 3: try: with urlopen(dotnetInstallScriptUrl) as response: if "html" in response.info()['Content-Type']: count = count + 1 sleep(1) # sleep one second continue with open(dotnetInstallScriptPath, 'wb') as outfile: outfile.write(response.read()) break except Exception as ex: count = count + 1 print(f"Attempt#{count} : {str(ex)}") sleep(1) continue if count == 3: getLogger().error("Fatal error: could not download dotnet-install script") raise Exception("Fatal error: could not download dotnet-install script") if platform != 'win32': chmod(dotnetInstallScriptPath, S_IRWXU) dotnetInstallInterpreter = [ 'powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass', dotnetInstallScriptPath ] if platform == 'win32' else [dotnetInstallScriptPath] # If Version is supplied, pull down the specified version common_cmdline_args = dotnetInstallInterpreter + [ '-InstallDir', install_dir, '-Architecture', architecture ] # Install Runtime/SDKs if versions: for version in versions: cmdline_args = common_cmdline_args + ['-Version', version] RunCommand(cmdline_args, verbose=verbose, retry=1).run( get_repo_root_path() ) # Only check channels if versions are not supplied. # When we supply a version, but still pull down with -Channel, we will use # whichever sdk is newer. So if we are trying to check an older version, # or if there is a new version between when we start a run and when we actually # run, we will be testing the "wrong" version, ie, not the version we specified. if (not versions) and channels: for channel in channels: cmdline_args = common_cmdline_args + ['-Channel', channel] RunCommand(cmdline_args, verbose=verbose, retry=1).run( get_repo_root_path() ) # Set DotNet Cli environment variables. environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' environ['UseSharedCompilation'] = 'false' environ['DOTNET_ROOT'] = install_dir # Add installed dotnet cli to PATH environ["PATH"] = install_dir + pathsep + environ["PATH"] # If we have copied dotnet from a different machine, then it may not be # marked as executable. Fix this. if platform != 'win32': chmod(path.join(install_dir, 'dotnet'), S_IRWXU)
def install(architecture: str, channels: list, versions: str, verbose: bool, install_dir: str = None) -> None: ''' Downloads dotnet cli into the tools folder. ''' __log_script_header("Downloading DotNet Cli") if not install_dir: install_dir = __get_directory(architecture) if not path.exists(install_dir): makedirs(install_dir) getLogger().info("DotNet Install Path: '%s'", install_dir) # Download appropriate dotnet install script dotnetInstallScriptExtension = '.ps1' if platform == 'win32' else '.sh' dotnetInstallScriptName = 'dotnet-install' + dotnetInstallScriptExtension url = 'https://dot.net/v1/' dotnetInstallScriptUrl = url + dotnetInstallScriptName dotnetInstallScriptPath = path.join(install_dir, dotnetInstallScriptName) getLogger().info('Downloading %s', dotnetInstallScriptUrl) count = 0 while count < 3: try: with urlopen(dotnetInstallScriptUrl, context=ssl._create_unverified_context()) as response: if "html" in response.info()['Content-Type']: count = count + 1 sleep(1) # sleep one second continue with open(dotnetInstallScriptPath, 'wb') as outfile: outfile.write(response.read()) break except Exception: count = count + 1 sleep(1) continue if count == 3: getLogger().error( "Fatal error: could not download dotnet-install script") raise Exception( "Fatal error: could not download dotnet-install script") if platform != 'win32': chmod(dotnetInstallScriptPath, S_IRWXU) dotnetInstallInterpreter = [ 'powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass', dotnetInstallScriptPath ] if platform == 'win32' else [dotnetInstallScriptPath] # If Version is supplied, pull down the specified version common_cmdline_args = dotnetInstallInterpreter + [ '-InstallDir', install_dir, '-Architecture', architecture ] # Install Runtime/SDKs if versions: for version in versions: cmdline_args = common_cmdline_args + ['-Version', version] RunCommand(cmdline_args, verbose=verbose, retry=1).run(get_repo_root_path()) # Only check channels if versions are not supplied. # When we supply a version, but still pull down with -Channel, we will use # whichever sdk is newer. So if we are trying to check an older version, # or if there is a new version between when we start a run and when we actually # run, we will be testing the "wrong" version, ie, not the version we specified. if (not versions) and channels: for channel in channels: cmdline_args = common_cmdline_args + [ '-Channel', ChannelMap.get_branch(channel) ] if ChannelMap.get_quality_from_channel(channel) is not None: cmdline_args += [ '-Quality', ChannelMap.get_quality_from_channel(channel) ] RunCommand(cmdline_args, verbose=verbose, retry=1).run(get_repo_root_path()) setup_dotnet(install_dir)
def __main(args: list) -> int: validate_supported_runtime() args = __process_arguments(args) verbose = not args.quiet setup_loggers(verbose=verbose) if not args.frameworks: raise Exception("Framework version (-f) must be specified.") target_framework_monikers = dotnet \ .FrameworkAction \ .get_target_framework_monikers(args.frameworks) # Acquire necessary tools (dotnet) init_tools(architecture=args.architecture, dotnet_versions=args.dotnet_versions, target_framework_monikers=target_framework_monikers, verbose=verbose) # WORKAROUND # The MicroBenchmarks.csproj targets .NET Core 2.1, 3.0, 3.1 and 5.0 # to avoid a build failure when using older frameworks (error NETSDK1045: # The current .NET SDK does not support targeting .NET Core $XYZ) # we set the TFM to what the user has provided. os.environ['PERFLAB_TARGET_FRAMEWORKS'] = ';'.join( target_framework_monikers) # dotnet --info dotnet.info(verbose=verbose) BENCHMARKS_CSPROJ = dotnet.CSharpProject(project=args.csprojfile, bin_directory=args.bin_directory) if not args.run_only: # .NET micro-benchmarks # Restore and build micro-benchmarks micro_benchmarks.build(BENCHMARKS_CSPROJ, args.configuration, target_framework_monikers, args.incremental, verbose) # Run micro-benchmarks if not args.build_only: upload_container = UPLOAD_CONTAINER try: for framework in args.frameworks: micro_benchmarks.run(BENCHMARKS_CSPROJ, args.configuration, framework, verbose, args) globpath = os.path.join( get_artifacts_directory() if not args.bdn_artifacts else args.bdn_artifacts, '**', '*perf-lab-report.json') except CalledProcessError: upload_container = 'failedresults' globpath = os.path.join( get_artifacts_directory() if not args.bdn_artifacts else args.bdn_artifacts, 'FailureReporter', 'failure-report.json') cmdline = ['FailureReporting.exe', globpath] reporterpath = os.path.join(helixpayload(), 'FailureReporter') if not os.path.exists(reporterpath): raise FileNotFoundError RunCommand(cmdline, verbose=True).run(reporterpath) dotnet.shutdown_server(verbose) if args.upload_to_perflab_container: import upload upload.upload(globpath, upload_container, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def install(architecture: str, channels: list, versions: str, verbose: bool, install_dir: str = None) -> None: ''' Downloads dotnet cli into the tools folder. ''' __log_script_header("Downloading DotNet Cli") if not install_dir: install_dir = __get_directory(architecture) if not path.exists(install_dir): makedirs(install_dir) getLogger().info("DotNet Install Path: '%s'", install_dir) # Download appropriate dotnet install script dotnetInstallScriptExtension = '.ps1' if platform == 'win32' else '.sh' dotnetInstallScriptName = 'dotnet-install' + dotnetInstallScriptExtension url = 'https://dot.net/v1/' dotnetInstallScriptUrl = url + dotnetInstallScriptName dotnetInstallScriptPath = path.join(install_dir, dotnetInstallScriptName) getLogger().info('Downloading %s', dotnetInstallScriptUrl) urlretrieve(dotnetInstallScriptUrl, dotnetInstallScriptPath) if platform != 'win32': chmod(dotnetInstallScriptPath, S_IRWXU) dotnetInstallInterpreter = [ 'powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass', dotnetInstallScriptPath ] if platform == 'win32' else [dotnetInstallScriptPath] # If Version is supplied, pull down the specified version common_cmdline_args = dotnetInstallInterpreter + [ '-InstallDir', install_dir, '-Architecture', architecture ] # Install Runtime/SDKs if versions: for version in versions: cmdline_args = common_cmdline_args + ['-Version', version] RunCommand(cmdline_args, verbose=verbose).run(get_repo_root_path()) # Only check channels if versions are not supplied. # When we supply a version, but still pull down with -Channel, we will use # whichever sdk is newer. So if we are trying to check an older version, # or if there is a new version between when we start a run and when we actually # run, we will be testing the "wrong" version, ie, not the version we specified. if (not versions) and channels: for channel in channels: cmdline_args = common_cmdline_args + ['-Channel', channel] RunCommand(cmdline_args, verbose=verbose).run(get_repo_root_path()) # Set DotNet Cli environment variables. environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' environ['UseSharedCompilation'] = 'false' environ['DOTNET_ROOT'] = install_dir # Add installed dotnet cli to PATH environ["PATH"] = install_dir + pathsep + environ["PATH"] # If we have copied dotnet from a different machine, it will not be executable. Fix this if platform != 'win32': chmod(path.join(install_dir, 'dotnet'), S_IRWXU)
def build(self, configuration: str, verbose: bool, packages_path: str, target_framework_monikers: list = None, output_to_bindir: bool = False, runtime_identifier: str = None, *args) -> None: '''Calls dotnet to build the specified project.''' if not target_framework_monikers: # Build all supported frameworks. cmdline = [ 'dotnet', 'build', self.csproj_file, '--configuration', configuration, '--no-restore', "/p:NuGetPackageRoot={}".format(packages_path), '/p:UseSharedCompilation=false', '/p:BuildInParallel=false', '/m:1', ] if output_to_bindir: cmdline = cmdline + ['--output', self.__bin_directory] if runtime_identifier: cmdline = cmdline + ['--runtime', runtime_identifier] if args: cmdline = cmdline + list(args) RunCommand(cmdline, verbose=verbose).run(self.working_directory) else: # Only build specified frameworks for target_framework_moniker in target_framework_monikers: cmdline = [ 'dotnet', 'build', self.csproj_file, '--configuration', configuration, '--framework', target_framework_moniker, '--no-restore', "/p:NuGetPackageRoot={}".format(packages_path), '/p:UseSharedCompilation=false', '/p:BuildInParallel=false', '/m:1', ] if output_to_bindir: cmdline = cmdline + ['--output', self.__bin_directory] if runtime_identifier: cmdline = cmdline + ['--runtime', runtime_identifier] if args: cmdline = cmdline + list(args) RunCommand(cmdline, verbose=verbose).run(self.working_directory)
def run(self): ''' Runs the specified scenario ''' self.parseargs() if self.testtype == const.INNERLOOP: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.INNERLOOPMSBUILD: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.DOTNETWATCH: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH], apptorun='dotnet', appargs='watch -v', innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) self.traits.add_traits(workingdir=const.APPDIR) startup.runtests(self.traits) if self.testtype == const.STARTUP: startup = StartupWrapper() self.traits.add_traits( overwrite=False, environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '', scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.STARTUP], apptorun=publishedexe(self.traits.exename), ) startup.runtests(self.traits) elif self.testtype == const.SDK: startup = StartupWrapper() envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0' envlistcleanbuild = ';'.join( ['MSBUILDDISABLENODEREUSE=1', envlistbuild]) # clean build if self.sdktype == const.CLEAN_BUILD: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD), apptorun=const.DOTNET, appargs='build', iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR, environmentvariables=envlistcleanbuild, ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # build(no changes) if self.sdktype == const.BUILD_NO_CHANGE: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE), apptorun=const.DOTNET, appargs='build', workingdir=const.APPDIR, environmentvariables=envlistbuild) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # new console if self.sdktype == const.NEW_CONSOLE: self.traits.add_traits( overwrite=False, appargs='new console', apptorun=const.DOTNET, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE), iterationsetup=pythoncommand(), setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN: startup = StartupWrapper() crossgenexe = 'crossgen%s' % extension() crossgenargs = self.crossgen_arguments.get_crossgen_command_line() coreroot = self.crossgen_arguments.coreroot scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME, workingdir=coreroot, appargs=' '.join(crossgenargs)) self.traits.add_traits( overwrite=False, scenarioname='Crossgen Throughput - %s' % scenario_filename, scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename), apptorun='%s\%s' % (coreroot, crossgenexe), ) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN2: startup = StartupWrapper() scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) crossgen2args = self.crossgen_arguments.get_crossgen2_command_line( ) compiletype = self.crossgen_arguments.crossgen2_compiletype() scenarioname = 'Crossgen2 Throughput - %s - %s' % ( compiletype, scenario_filename) if self.crossgen_arguments.singlethreaded: scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % ( compiletype, scenario_filename) if compiletype == const.CROSSGEN2_COMPOSITE: self.traits.add_traits(overwrite=True, skipprofile='true') self.traits.add_traits( overwrite=True, startupmetric=const.STARTUP_CROSSGEN2, workingdir=self.crossgen_arguments.coreroot, appargs='%s %s' % (os.path.join( 'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args))) self.traits.add_traits( overwrite=False, scenarioname=scenarioname, apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()), environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events ) startup.runtests(self.traits) elif self.testtype == const.DEVICESTARTUP: # ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc # Regex used to split the response from starting the activity and saving each value #Example: # Starting: Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity } # Status: ok # LaunchState: COLD # Activity: net.dot.HelloAndroid/net.dot.MainActivity # TotalTime: 241 # WaitTime: 242 # Complete # Saves: [Intent { cmp=net.dot.HelloAndroid/net.dot.MainActivity }, ok, COLD, net.dot.HelloAndroid/net.dot.MainActivity, 241, 242] # Split results (start at 0) (List is Starting (Intent activity), Status (ok...), LaunchState ([HOT, COLD, WARM]), Activity (started activity name), TotalTime(toFrameOne), WaitTime(toFullLoad)) runSplitRegex = ":\s(.+)" screenWasOff = False getLogger().info("Clearing potential previous run nettraces") for file in glob.glob( os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')): if exists(file): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb'] adb = RunCommand(cmdline, verbose=True) adb.run() # Do not remove, XHarness install seems to fail without an adb command called before the xharness command getLogger().info("Preparing ADB") cmdline = [adb.stdout.strip(), 'shell', 'wm', 'size'] RunCommand(cmdline, verbose=True).run() # Get animation values getLogger().info("Getting animation values") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] window_animation_scale_cmd = RunCommand(cmdline, verbose=True) window_animation_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] transition_animation_scale_cmd = RunCommand(cmdline, verbose=True) transition_animation_scale_cmd.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] animator_duration_scale_cmd = RunCommand(cmdline, verbose=True) animator_duration_scale_cmd.run() getLogger().info( f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}" ) # Make sure animations are set to 1 or disabled getLogger().info("Setting animation values") if (self.animationsdisabled): animationValue = 0 else: animationValue = 1 cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'transition_animation_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'animator_duration_scale', str(animationValue) ] RunCommand(cmdline, verbose=True).run() # Check for success getLogger().info("Getting animation values to verify it worked") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] windowSetValue = RunCommand(cmdline, verbose=True) windowSetValue.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] transitionSetValue = RunCommand(cmdline, verbose=True) transitionSetValue.run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] animatorSetValue = RunCommand(cmdline, verbose=True) animatorSetValue.run() if (int(windowSetValue.stdout.strip()) != animationValue or int(transitionSetValue.stdout.strip()) != animationValue or int(animatorSetValue.stdout.strip()) != animationValue): # Setting the values didn't work, error out getLogger().exception( f"Failed to set animation values to {animationValue}.") exit(-1) else: getLogger().info( f"Animation values successfully set to {animationValue}.") installCmd = xharnesscommand() + [ self.devicetype, 'install', '--app', self.packagepath, '--package-name', self.packagename, '-o', const.TRACEDIR, '-v' ] RunCommand(installCmd, verbose=True).run() getLogger().info("Completed install, running shell.") cmdline = [ adb.stdout.strip(), 'shell', f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' ] getActivity = RunCommand(cmdline, verbose=True) getActivity.run() getLogger().info(f"Target Activity {getActivity.stdout}") # More setup stuff checkScreenOnCmd = [ adb.stdout.strip(), 'shell', f'dumpsys input_method | grep mInteractive' ] checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() keyInputCmd = [adb.stdout.strip(), 'shell', 'input', 'keyevent'] if ("mInteractive=false" in checkScreenOn.stdout): # Turn on the screen to make interactive and see if it worked getLogger().info("Screen was off, turning on.") screenWasOff = True RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key RunCommand(keyInputCmd + ['82'], verbose=True).run( ) # Unlock the screen with menu key (only works if it is not a password lock) checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() if ("mInteractive=false" in checkScreenOn.stdout): getLogger().exception("Failed to make screen interactive.") exit(-1) # Actual testing some run stuff getLogger().info("Test run to check if permissions are needed") activityname = getActivity.stdout startAppCmd = [ adb.stdout.strip(), 'shell', 'am', 'start-activity', '-W', '-n', activityname ] testRun = RunCommand(startAppCmd, verbose=True) testRun.run() testRunStats = re.findall( runSplitRegex, testRun.stdout ) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") stopAppCmd = [ adb.stdout.strip(), 'shell', 'am', 'force-stop', self.packagename ] RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['66'], verbose=True).run( ) # Press enter to close main perm screen time.sleep(1) RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button time.sleep(1) RunCommand(keyInputCmd + ['66'], verbose=True).run( ) # Press enter to close out of second screen time.sleep(1) # Check to make sure it worked testRun = RunCommand(startAppCmd, verbose=True) testRun.run() testRunStats = re.findall(runSplitRegex, testRun.stdout) getLogger().info(f"Test run activity: {testRunStats[3]}") RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[ 3]: getLogger().exception( "Failed to get past permission screen, run locally to see if enough next button presses were used." ) exit(-1) allResults = [] for i in range(self.startupiterations): startStats = RunCommand(startAppCmd, verbose=True) startStats.run() RunCommand(stopAppCmd, verbose=True).run() allResults.append( startStats.stdout ) # Save results (List is Intent, Status, LaunchState Activity, TotalTime, WaitTime) time.sleep(3) # Delay in seconds for ensuring a cold start getLogger().info("Stopping App for uninstall") RunCommand(stopAppCmd, verbose=True).run() getLogger().info("Uninstalling app") uninstallAppCmd = xharnesscommand() + [ 'android', 'uninstall', '--package-name', self.packagename ] RunCommand(uninstallAppCmd, verbose=True).run() # Reset animation values getLogger().info("Resetting animation values to pretest values") cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'window_animation_scale', window_animation_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'transition_animation_scale', transition_animation_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'settings', 'put', 'global', 'animator_duration_scale', animator_duration_scale_cmd.stdout.strip() ] RunCommand(cmdline, verbose=True).run() if screenWasOff: RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off # Create traces to store the data so we can keep the current general parse trace flow getLogger().info(f"Logs: \n{allResults}") os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True) traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w") for result in allResults: traceFile.write(result) traceFile.close() startup = StartupWrapper() self.traits.add_traits( overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) startup.parsetraces(self.traits) elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None if not builtdir: builtdir = const.BINDIR if os.path.exists( const.BINDIR) else None if not (self.dirs or builtdir): raise Exception( "Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR)) sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)
def runtests(self, traits: TestTraits): ''' Runs tests through startup ''' # make sure required arguments are present for key in ['apptorun', 'startupmetric', 'guiapp']: if not getattr(traits, key): raise Exception('startup tests require %s' % key) defaultiterations = '1' if runninginlab() and not uploadtokenpresent() else '5' # only run 1 iteration for PR-triggered build # required arguments & optional arguments with default values startup_args = [ self.startuppath, '--app-exe', traits.apptorun, '--metric-type', traits.startupmetric, '--trace-name', '%s_startup' % (traits.scenarioname or '%s_%s' % (traits.exename,traits.scenariotypename)), '--gui-app', traits.guiapp, '--process-will-exit', (traits.processwillexit or 'true'), '--iterations', '%s' % (traits.iterations or defaultiterations), '--timeout', '%s' % (traits.timeout or '50'), '--warmup', '%s' % (traits.warmup or 'true'), '--working-dir', '%s' % (traits.workingdir or sys.path[0]), '--report-json-path', self.reportjson, '--trace-directory', TRACEDIR ] # optional arguments without default values if traits.scenarioname: startup_args.extend(['--scenario-name', traits.scenarioname]) if traits.appargs: startup_args.extend(['--app-args', traits.appargs]) if traits.environmentvariables: startup_args.extend(['--environment-variables', traits.environmentvariables]) if traits.iterationsetup: startup_args.extend(['--iteration-setup', traits.iterationsetup]) if traits.setupargs: startup_args.extend(['--setup-args', traits.setupargs]) if traits.iterationcleanup: startup_args.extend(['--iteration-cleanup', traits.iterationcleanup]) if traits.cleanupargs: startup_args.extend(['--cleanup-args', traits.cleanupargs]) if traits.measurementdelay: startup_args.extend(['--measurement-delay', traits.measurementdelay]) if traits.skipprofile: startup_args.extend(['--skip-profile-iteration']) if traits.innerloopcommand: startup_args.extend(['--inner-loop-command', traits.innerloopcommand]) if traits.innerloopcommandargs: startup_args.extend(['--inner-loop-command-args', traits.innerloopcommandargs]) if traits.runwithoutexit: startup_args.extend(['--run-without-exit', traits.runwithoutexit]) if traits.hotreloaditers: startup_args.extend(['--hot-reload-iters', traits.hotreloaditers]) if traits.skipmeasurementiteration: startup_args.extend(['--skip-measurement-iteration', traits.skipmeasurementiteration]) upload_container = UPLOAD_CONTAINER try: RunCommand(startup_args, verbose=True).run() except CalledProcessError: getLogger().info("Run failure registered") # rethrow the original exception raise if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload_code = upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI) getLogger().info("Startup Upload Code: " + str(upload_code)) if upload_code != 0: sys.exit(upload_code)
def shutdown_server(verbose: bool) -> None: ''' Shuts down the dotnet server ''' cmdline = ['dotnet', 'build-server', 'shutdown'] RunCommand(cmdline, verbose=verbose).run(get_repo_root_path())
def run(self): ''' Runs the specified scenario ''' self.parseargs() if self.testtype == const.INNERLOOP: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOP], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.INNERLOOPMSBUILD: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.INNERLOOPMSBUILD], apptorun='dotnet', appargs='run --project %s' % appfolder(self.traits.exename, self.traits.projext), innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) startup.runtests(self.traits) if self.testtype == const.DOTNETWATCH: startup = StartupWrapper() self.traits.add_traits( scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.DOTNETWATCH], apptorun='dotnet', appargs='watch -v', innerloopcommand=pythoncommand(), iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE)) self.traits.add_traits(workingdir=const.APPDIR) startup.runtests(self.traits) if self.testtype == const.STARTUP: startup = StartupWrapper() self.traits.add_traits( overwrite=False, environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '', scenarioname=self.scenarioname, scenariotypename=const.SCENARIO_NAMES[const.STARTUP], apptorun=publishedexe(self.traits.exename), ) startup.runtests(self.traits) elif self.testtype == const.SDK: startup = StartupWrapper() envlistbuild = 'DOTNET_MULTILEVEL_LOOKUP=0' envlistcleanbuild = ';'.join( ['MSBUILDDISABLENODEREUSE=1', envlistbuild]) # clean build if self.sdktype == const.CLEAN_BUILD: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.CLEAN_BUILD), apptorun=const.DOTNET, appargs='build', iterationsetup=pythoncommand(), setupargs='%s %s setup_build' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR, environmentvariables=envlistcleanbuild, ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # build(no changes) if self.sdktype == const.BUILD_NO_CHANGE: self.traits.add_traits( overwrite=False, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.BUILD_NO_CHANGE), apptorun=const.DOTNET, appargs='build', workingdir=const.APPDIR, environmentvariables=envlistbuild) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) # new console if self.sdktype == const.NEW_CONSOLE: self.traits.add_traits( overwrite=False, appargs='new console', apptorun=const.DOTNET, scenarioname=self.scenarioname, scenariotypename='%s_%s' % (const.SCENARIO_NAMES[const.SDK], const.NEW_CONSOLE), iterationsetup=pythoncommand(), setupargs='%s %s setup_new' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), iterationcleanup=pythoncommand(), cleanupargs='%s %s cleanup' % ('-3' if iswin() else '', const.ITERATION_SETUP_FILE), workingdir=const.APPDIR) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN: startup = StartupWrapper() crossgenexe = 'crossgen%s' % extension() crossgenargs = self.crossgen_arguments.get_crossgen_command_line() coreroot = self.crossgen_arguments.coreroot scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) self.traits.add_traits(overwrite=True, startupmetric=const.STARTUP_PROCESSTIME, workingdir=coreroot, appargs=' '.join(crossgenargs)) self.traits.add_traits( overwrite=False, scenarioname='Crossgen Throughput - %s' % scenario_filename, scenariotypename='%s - %s' % (const.SCENARIO_NAMES[const.CROSSGEN], scenario_filename), apptorun='%s\%s' % (coreroot, crossgenexe), ) startup.runtests(self.traits) elif self.testtype == const.CROSSGEN2: startup = StartupWrapper() scenario_filename = self.crossgen_arguments.crossgen2_scenario_filename( ) crossgen2args = self.crossgen_arguments.get_crossgen2_command_line( ) compiletype = self.crossgen_arguments.crossgen2_compiletype() scenarioname = 'Crossgen2 Throughput - %s - %s' % ( compiletype, scenario_filename) if self.crossgen_arguments.singlethreaded: scenarioname = 'Crossgen2 Throughput - Single Threaded - %s - %s' % ( compiletype, scenario_filename) if compiletype == const.CROSSGEN2_COMPOSITE: self.traits.add_traits(overwrite=True, skipprofile='true') self.traits.add_traits( overwrite=True, startupmetric=const.STARTUP_CROSSGEN2, workingdir=self.crossgen_arguments.coreroot, appargs='%s %s' % (os.path.join( 'crossgen2', 'crossgen2.dll'), ' '.join(crossgen2args))) self.traits.add_traits( overwrite=False, scenarioname=scenarioname, apptorun=os.path.join(self.crossgen_arguments.coreroot, 'corerun%s' % extension()), environmentvariables='COMPlus_EnableEventLog=1' if not iswin() else '' # turn on clr user events ) startup.runtests(self.traits) elif self.testtype == const.DEVICESTARTUP: getLogger().info("Clearing potential previous run nettraces") for file in glob.glob( os.path.join(const.TRACEDIR, 'PerfTest', 'trace*.nettrace')): if exists(file): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb'] adb = RunCommand(cmdline, verbose=True) adb.run() cmdline = [ adb.stdout.strip(), 'shell', 'mkdir', '-p', '/sdcard/PerfTest' ] RunCommand(cmdline, verbose=True).run() cmdline = xharnesscommand() + [ self.devicetype, 'install', '--app', self.packagepath, '--package-name', self.packagename, '-o', const.TRACEDIR, '-v' ] RunCommand(cmdline, verbose=True).run() for i in range(self.startupiterations): cmdline = xharnesscommand() + [ 'android', 'run', '-o', const.TRACEDIR, '--package-name', self.packagename, '-v', '--arg=env:COMPlus_EnableEventPipe=1', '--arg=env:COMPlus_EventPipeOutputStreaming=1', '--arg=env:COMPlus_EventPipeOutputPath=/sdcard/PerfTest/trace%s.nettrace' % (i + 1), '--arg=env:COMPlus_EventPipeCircularMB=10', '--arg=env:COMPlus_EventPipeConfig=Microsoft-Windows-DotNETRuntime:10:5', '--expected-exit-code', self.expectedexitcode, '--dev-out', '/sdcard/PerfTest/' ] RunCommand(cmdline, verbose=True).run() cmdline = xharnesscommand() + [ 'android', 'uninstall', '--package-name', self.packagename ] RunCommand(cmdline, verbose=True).run() cmdline = [ adb.stdout.strip(), 'shell', 'rm', '-r', '/sdcard/PerfTest' ] RunCommand(cmdline, verbose=True).run() startup = StartupWrapper() self.traits.add_traits( overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='trace*.nettrace', scenarioname='Device Startup - Android %s' % (self.packagename)) startup.parsetraces(self.traits) elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None if not builtdir: builtdir = const.BINDIR if os.path.exists( const.BINDIR) else None if not (self.dirs or builtdir): raise Exception( "Dirs was not passed in and neither %s nor %s exist" % (const.PUBDIR, const.BINDIR)) sod.runtests(scenarioname=self.scenarioname, dirs=self.dirs or builtdir, artifact=self.traits.artifact)
def upload_files(self, search_path: str): self.download_azcopy() cmdline = [ self.exe_path(), 'copy', search_path, self.get_upload_url(), '--recursive=true' ] RunCommand(cmdline, verbose=self.verbose).run()