def parsetraces(self, traits: TestTraits): directory = TRACEDIR if traits.tracefolder: directory = TRACEDIR + '/' + traits.tracefolder getLogger().info("Parse Directory: " + directory) startup_args = [ self.startuppath, '--app-exe', traits.apptorun, '--parse-only', '--metric-type', traits.startupmetric, '--trace-name', traits.tracename, '--report-json-path', self.reportjson, '--trace-directory', directory ] if traits.scenarioname: startup_args.extend(['--scenario-name', traits.scenarioname]) upload_container = UPLOAD_CONTAINER try: RunCommand(startup_args, verbose=True).run() except CalledProcessError: getLogger().info("Run failure registered") # rethrow the original exception raise if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, scenarioname, dirs, artifact=None): ''' Runs tests through sod tool ''' if not os.path.exists(TRACEDIR): os.mkdir(TRACEDIR) reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json') sod_args = [ self.sodexe, '--report-json-path', reportjson, '--scenario-name', (scenarioname or "Empty Scenario Name"), '--dirs' ] sod_args += dirs.split(';') RunCommand(sod_args, verbose=True).run() if artifact: if not os.path.exists(artifact): raise FileNotFoundError(f'Artifact {artifact} is not found.') else: copy(artifact, TRACEDIR) if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload_code = upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI) getLogger().info("SoD Upload Code: " + str(upload_code)) if upload_code != 0: sys.exit(upload_code)
def runtests(self, traits: TestTraits): ''' Runs tests through startup ''' # make sure required arguments are present for key in ['apptorun', 'startupmetric', 'guiapp']: if not getattr(traits, key): raise Exception('startup tests require %s' % key) reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json') defaultiterations = '1' if runninginlab() and not uploadtokenpresent( ) else '5' # only run 1 iteration for PR-triggered build # required arguments & optional arguments with default values startup_args = [ self.startuppath, '--app-exe', traits.apptorun, '--metric-type', traits.startupmetric, '--trace-name', '%s_startup' % (traits.scenarioname or '%s_%s' % (traits.exename, traits.scenariotypename)), '--gui-app', traits.guiapp, '--process-will-exit', (traits.processwillexit or 'true'), '--iterations', '%s' % (traits.iterations or defaultiterations), '--timeout', '%s' % (traits.timeout or '50'), '--warmup', '%s' % (traits.warmup or 'true'), '--working-dir', '%s' % (traits.workingdir or sys.path[0]), '--report-json-path', reportjson, '--trace-directory', TRACEDIR ] # optional arguments without default values if traits.scenarioname: startup_args.extend(['--scenario-name', traits.scenarioname]) if traits.appargs: startup_args.extend(['--app-args', traits.appargs]) if traits.environmentvariables: startup_args.extend( ['--environment-variables', traits.environmentvariables]) if traits.iterationsetup: startup_args.extend(['--iteration-setup', traits.iterationsetup]) if traits.setupargs: startup_args.extend(['--setup-args', traits.setupargs]) if traits.iterationcleanup: startup_args.extend( ['--iteration-cleanup', traits.iterationcleanup]) if traits.cleanupargs: startup_args.extend(['--cleanup-args', traits.cleanupargs]) if traits.measurementdelay: startup_args.extend( ['--measurement-delay', traits.measurementdelay]) if traits.skipprofile: startup_args.extend(['--skip-profile-iteration']) RunCommand(startup_args, verbose=True).run() if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, apptorun: str, **kwargs): ''' Runs tests through startup ''' for key in ['startupmetric', 'guiapp']: if not kwargs[key]: raise Exception('startup tests require %s' % key) reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json') defaultiterations = '1' if runninginlab() and not uploadtokenpresent( ) else '5' # only run 1 iteration for PR-triggered build startup_args = [ self.startupexe, '--app-exe', apptorun, '--metric-type', kwargs['startupmetric'], '--trace-file-name', '%s_startup.etl' % (kwargs['scenarioname'] or '%s_%s' % (kwargs['exename'], kwargs['scenariotypename'])), '--process-will-exit', (kwargs['processwillexit'] or 'true'), '--iterations', '%s' % (kwargs['iterations'] or defaultiterations), '--timeout', '%s' % (kwargs['timeout'] or '50'), '--warmup', '%s' % (kwargs['warmup'] or 'true'), '--gui-app', kwargs['guiapp'], '--working-dir', '%s' % (kwargs['workingdir'] or sys.path[0]), '--report-json-path', reportjson, '--trace-directory', TRACEDIR ] # optional arguments if kwargs['scenarioname']: startup_args.extend(['--scenario-name', kwargs['scenarioname']]) if kwargs['appargs']: startup_args.extend(['--app-args', kwargs['appargs']]) if kwargs['environmentvariables']: startup_args.extend( ['--environment-variables', kwargs['environmentvariables']]) if kwargs['iterationsetup']: startup_args.extend( ['--iteration-setup', kwargs['iterationsetup']]) if kwargs['setupargs']: startup_args.extend(['--setup-args', kwargs['setupargs']]) if kwargs['iterationcleanup']: startup_args.extend( ['--iteration-cleanup', kwargs['iterationcleanup']]) if kwargs['cleanupargs']: startup_args.extend(['--cleanup-args', kwargs['cleanupargs']]) if kwargs['measurementdelay']: startup_args.extend( ['--measurement-delay', kwargs['measurementdelay']]) RunCommand(startup_args, verbose=True).run() if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, apptorun: str, **kwargs): ''' Runs tests through startup ''' for key in ['startupmetric', 'guiapp']: if not kwargs[key]: raise Exception('startup tests require %s' % key) reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json') startup_args = [ self.startupexe, '--app-exe', apptorun, '--metric-type', kwargs['startupmetric'], '--scenario-name', "%s - %s" % (kwargs['scenarioname'], kwargs['scenariotypename']), '--trace-file-name', '%s_startup.etl' % kwargs['exename'], '--process-will-exit', 'true', # ??? '--iterations', '%s' % (kwargs['iterations'] or '5'), '--timeout', '%s' % (kwargs['timeout'] or '20'), '--warmup', '%s' % (kwargs['warmup'] or 'true'), '--gui-app', kwargs['guiapp'], '--working-dir', sys.path[0], '--report-json-path', reportjson, '--trace-directory', TRACEDIR ] RunCommand(startup_args, verbose=True).run() if runninginlab() and uploadtokenpresent(): import upload copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) upload.upload(reportjson, UPLOAD_CONTAINER, None, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, scenarioname, dirs): ''' Runs tests through sod tool ''' if not os.path.exists(TRACEDIR): os.mkdir(TRACEDIR) reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json') sod_args = [ self.sodexe, '--report-json-path', reportjson, '--scenario-name', (scenarioname or "Empty Scenario Name"), '--dirs', dirs ] RunCommand(sod_args, verbose=True).run() if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, traits: TestTraits): ''' Runs tests through startup ''' # make sure required arguments are present for key in ['apptorun', 'startupmetric', 'guiapp']: if not getattr(traits, key): raise Exception('startup tests require %s' % key) reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json') defaultiterations = '1' if runninginlab() and not uploadtokenpresent( ) else '5' # only run 1 iteration for PR-triggered build # required arguments & optional arguments with default values startup_args = [ self.startuppath, '--app-exe', traits.apptorun, '--metric-type', traits.startupmetric, '--trace-name', '%s_startup' % (traits.scenarioname or '%s_%s' % (traits.exename, traits.scenariotypename)), '--gui-app', traits.guiapp, '--process-will-exit', (traits.processwillexit or 'true'), '--iterations', '%s' % (traits.iterations or defaultiterations), '--timeout', '%s' % (traits.timeout or '50'), '--warmup', '%s' % (traits.warmup or 'true'), '--working-dir', '%s' % (traits.workingdir or sys.path[0]), '--report-json-path', reportjson, '--trace-directory', TRACEDIR ] # optional arguments without default values if traits.scenarioname: startup_args.extend(['--scenario-name', traits.scenarioname]) if traits.appargs: startup_args.extend(['--app-args', traits.appargs]) if traits.environmentvariables: startup_args.extend( ['--environment-variables', traits.environmentvariables]) if traits.iterationsetup: startup_args.extend(['--iteration-setup', traits.iterationsetup]) if traits.setupargs: startup_args.extend(['--setup-args', traits.setupargs]) if traits.iterationcleanup: startup_args.extend( ['--iteration-cleanup', traits.iterationcleanup]) if traits.cleanupargs: startup_args.extend(['--cleanup-args', traits.cleanupargs]) if traits.measurementdelay: startup_args.extend( ['--measurement-delay', traits.measurementdelay]) if traits.skipprofile: startup_args.extend(['--skip-profile-iteration']) if traits.innerloopcommand: startup_args.extend( ['--inner-loop-command', traits.innerloopcommand]) if traits.innerloopcommandargs: startup_args.extend( ['--inner-loop-command-args', traits.innerloopcommandargs]) if traits.runwithoutexit: startup_args.extend(['--run-without-exit', traits.runwithoutexit]) if traits.hotreloaditers: startup_args.extend(['--hot-reload-iters', traits.hotreloaditers]) if traits.skipmeasurementiteration: startup_args.extend([ '--skip-measurement-iteration', traits.skipmeasurementiteration ]) upload_container = UPLOAD_CONTAINER try: RunCommand(startup_args, verbose=True).run() except CalledProcessError: getLogger().info("Run failure registered") if runninginlab(): upload_container = 'failedresults' reportdir = os.path.join(TRACEDIR, 'FailureReporter') os.makedirs(reportdir) reportjson = os.path.join(os.getcwd(), reportdir, 'failure-report.json') cmdline = ["FailureReporting%s" % extension(), reportjson] reporterpath = os.path.join(helixpayload(), 'FailureReporter') if not os.path.exists(reporterpath): raise FileNotFoundError getLogger().info("Generating failure results at " + reportjson) RunCommand(cmdline, verbose=True).run(reporterpath) if runninginlab(): copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces')) if uploadtokenpresent(): import upload upload.upload(reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)