def runtests(self, traits: TestTraits):
'''
Runs tests through startup
'''
# make sure required arguments are present
for key in ['apptorun', 'startupmetric', 'guiapp']:
if not getattr(traits, key):
raise Exception('startup tests require %s' % key)
reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
defaultiterations = '1' if runninginlab() and not uploadtokenpresent(
) else '5' # only run 1 iteration for PR-triggered build
# required arguments & optional arguments with default values
startup_args = [
self.startuppath, '--app-exe', traits.apptorun, '--metric-type',
traits.startupmetric, '--trace-name',
'%s_startup' % (traits.scenarioname or '%s_%s' %
(traits.exename, traits.scenariotypename)),
'--gui-app', traits.guiapp, '--process-will-exit',
(traits.processwillexit or 'true'), '--iterations',
'%s' % (traits.iterations or defaultiterations), '--timeout',
'%s' % (traits.timeout or '50'), '--warmup',
'%s' % (traits.warmup or 'true'), '--working-dir',
'%s' % (traits.workingdir or sys.path[0]), '--report-json-path',
reportjson, '--trace-directory', TRACEDIR
]
# optional arguments without default values
if traits.scenarioname:
startup_args.extend(['--scenario-name', traits.scenarioname])
if traits.appargs:
startup_args.extend(['--app-args', traits.appargs])
if traits.environmentvariables:
startup_args.extend(
['--environment-variables', traits.environmentvariables])
if traits.iterationsetup:
startup_args.extend(['--iteration-setup', traits.iterationsetup])
if traits.setupargs:
startup_args.extend(['--setup-args', traits.setupargs])
if traits.iterationcleanup:
startup_args.extend(
['--iteration-cleanup', traits.iterationcleanup])
if traits.cleanupargs:
startup_args.extend(['--cleanup-args', traits.cleanupargs])
if traits.measurementdelay:
startup_args.extend(
['--measurement-delay', traits.measurementdelay])
if traits.skipprofile:
startup_args.extend(['--skip-profile-iteration'])
RunCommand(startup_args, verbose=True).run()
if runninginlab():
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
if uploadtokenpresent():
import upload
upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, apptorun: str, **kwargs):
'''
Runs tests through startup
'''
for key in ['startupmetric', 'guiapp']:
if not kwargs[key]:
raise Exception('startup tests require %s' % key)
reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
defaultiterations = '1' if runninginlab() and not uploadtokenpresent(
) else '5' # only run 1 iteration for PR-triggered build
startup_args = [
self.startupexe, '--app-exe', apptorun, '--metric-type',
kwargs['startupmetric'], '--trace-file-name',
'%s_startup.etl' %
(kwargs['scenarioname'] or '%s_%s' %
(kwargs['exename'], kwargs['scenariotypename'])),
'--process-will-exit', (kwargs['processwillexit']
or 'true'), '--iterations',
'%s' % (kwargs['iterations'] or defaultiterations), '--timeout',
'%s' % (kwargs['timeout'] or '50'), '--warmup',
'%s' % (kwargs['warmup'] or 'true'), '--gui-app', kwargs['guiapp'],
'--working-dir',
'%s' % (kwargs['workingdir'] or sys.path[0]), '--report-json-path',
reportjson, '--trace-directory', TRACEDIR
]
# optional arguments
if kwargs['scenarioname']:
startup_args.extend(['--scenario-name', kwargs['scenarioname']])
if kwargs['appargs']:
startup_args.extend(['--app-args', kwargs['appargs']])
if kwargs['environmentvariables']:
startup_args.extend(
['--environment-variables', kwargs['environmentvariables']])
if kwargs['iterationsetup']:
startup_args.extend(
['--iteration-setup', kwargs['iterationsetup']])
if kwargs['setupargs']:
startup_args.extend(['--setup-args', kwargs['setupargs']])
if kwargs['iterationcleanup']:
startup_args.extend(
['--iteration-cleanup', kwargs['iterationcleanup']])
if kwargs['cleanupargs']:
startup_args.extend(['--cleanup-args', kwargs['cleanupargs']])
if kwargs['measurementdelay']:
startup_args.extend(
['--measurement-delay', kwargs['measurementdelay']])
RunCommand(startup_args, verbose=True).run()
if runninginlab():
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
if uploadtokenpresent():
import upload
upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, scenarioname, dirs, artifact=None):
'''
Runs tests through sod tool
'''
if not os.path.exists(TRACEDIR):
os.mkdir(TRACEDIR)
reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
sod_args = [
self.sodexe, '--report-json-path', reportjson, '--scenario-name',
(scenarioname or "Empty Scenario Name"), '--dirs'
]
sod_args += dirs.split(';')
RunCommand(sod_args, verbose=True).run()
if artifact:
if not os.path.exists(artifact):
raise FileNotFoundError(f'Artifact {artifact} is not found.')
else:
copy(artifact, TRACEDIR)
if runninginlab():
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
if uploadtokenpresent():
import upload
upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def runtests(self, apptorun: str, **kwargs):
'''
Runs tests through startup
'''
for key in ['startupmetric', 'guiapp']:
if not kwargs[key]:
raise Exception('startup tests require %s' % key)
reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
startup_args = [
self.startupexe,
'--app-exe',
apptorun,
'--metric-type',
kwargs['startupmetric'],
'--scenario-name',
"%s - %s" % (kwargs['scenarioname'], kwargs['scenariotypename']),
'--trace-file-name',
'%s_startup.etl' % kwargs['exename'],
'--process-will-exit',
'true', # ???
'--iterations',
'%s' % (kwargs['iterations'] or '5'),
'--timeout',
'%s' % (kwargs['timeout'] or '20'),
'--warmup',
'%s' % (kwargs['warmup'] or 'true'),
'--gui-app',
kwargs['guiapp'],
'--working-dir',
sys.path[0],
'--report-json-path',
reportjson,
'--trace-directory',
TRACEDIR
]
RunCommand(startup_args, verbose=True).run()
if runninginlab() and uploadtokenpresent():
import upload
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
upload.upload(reportjson, UPLOAD_CONTAINER, None, UPLOAD_TOKEN_VAR,
UPLOAD_STORAGE_URI)
def runtests(self, scenarioname, dirs):
'''
Runs tests through sod tool
'''
if not os.path.exists(TRACEDIR):
os.mkdir(TRACEDIR)
reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
sod_args = [
self.sodexe, '--report-json-path', reportjson, '--scenario-name',
(scenarioname or "Empty Scenario Name"), '--dirs', dirs
]
RunCommand(sod_args, verbose=True).run()
if runninginlab():
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
if uploadtokenpresent():
import upload
upload.upload(reportjson, UPLOAD_CONTAINER, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
