def __init__(self):
if helixpayload() and os.path.exists(os.path.join(helixpayload(), 'SOD')):
self._setsodpath(os.path.join(helixpayload(), 'SOD'))
elif helixworkitempayload() and os.path.exists(os.path.join(helixworkitempayload(), 'SOD')):
self._setsodpath(os.path.join(helixworkitempayload(), 'SOD'))
else:
relpath = os.path.join(get_artifacts_directory(), 'SOD')
sodproj = os.path.join('..',
'..',
'tools',
'ScenarioMeasurement',
'SizeOnDisk',
'SizeOnDisk.csproj')
sod = CSharpProject(CSharpProjFile(sodproj,
sys.path[0]),
os.path.join(os.path.dirname(sodproj),
os.path.join(get_artifacts_directory(), 'SOD')))
if not os.path.exists(relpath):
sod.restore(get_packages_directory(),
True,
getruntimeidentifier())
sod.publish('Release',
relpath,
True,
get_packages_directory(),
None,
getruntimeidentifier(),
None,
'--no-restore'
)
self._setsodpath(sod.bin_path)
def __init__(self):
startupdir = 'startup'
self.reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
if helixpayload() and os.path.exists(os.path.join(helixpayload(), startupdir)):
self._setstartuppath(os.path.join(helixpayload(), startupdir))
elif helixworkitempayload() and os.path.exists(os.path.join(helixworkitempayload(), startupdir)):
self._setstartuppath(os.path.join(helixworkitempayload(), startupdir))
else:
relpath = os.path.join(get_artifacts_directory(), startupdir)
startupproj = os.path.join('..',
'..',
'tools',
'ScenarioMeasurement',
'Startup',
'Startup.csproj')
startup = CSharpProject(CSharpProjFile(startupproj,
sys.path[0]),
os.path.join(os.path.dirname(startupproj),
os.path.join(get_artifacts_directory(), startupdir)))
if not os.path.exists(relpath):
startup.restore(get_packages_directory(),
True,
getruntimeidentifier())
startup.publish('Release',
relpath,
True,
get_packages_directory(),
None,
getruntimeidentifier(),
None,
'--no-restore'
)
self._setstartuppath(startup.bin_path)
def add_event_source(self, file: str, line: str, trace_statement: str):
'''
Adds a copy of the event source to the project and inserts the correct call
file: relative path to the root of the project (where the project file lives)
line: Exact line to insert trace statement after
trace_statement: Statement to insert
'''
projpath = os.path.dirname(self.project.csproj_file)
staticpath = os.path.join(get_repo_root_path(), "src", "scenarios", "staticdeps")
if helixpayload():
staticpath = os.path.join(helixpayload(), "staticdeps")
shutil.copyfile(os.path.join(staticpath, "PerfLab.cs"), os.path.join(projpath, "PerfLab.cs"))
filepath = os.path.join(projpath, file)
insert_after(filepath, line, trace_statement)
def runtests(self, traits: TestTraits):
'''
Runs tests through startup
'''
# make sure required arguments are present
for key in ['apptorun', 'startupmetric', 'guiapp']:
if not getattr(traits, key):
raise Exception('startup tests require %s' % key)
reportjson = os.path.join(TRACEDIR, 'perf-lab-report.json')
defaultiterations = '1' if runninginlab() and not uploadtokenpresent(
) else '5' # only run 1 iteration for PR-triggered build
# required arguments & optional arguments with default values
startup_args = [
self.startuppath, '--app-exe', traits.apptorun, '--metric-type',
traits.startupmetric, '--trace-name',
'%s_startup' % (traits.scenarioname or '%s_%s' %
(traits.exename, traits.scenariotypename)),
'--gui-app', traits.guiapp, '--process-will-exit',
(traits.processwillexit or 'true'), '--iterations',
'%s' % (traits.iterations or defaultiterations), '--timeout',
'%s' % (traits.timeout or '50'), '--warmup',
'%s' % (traits.warmup or 'true'), '--working-dir',
'%s' % (traits.workingdir or sys.path[0]), '--report-json-path',
reportjson, '--trace-directory', TRACEDIR
]
# optional arguments without default values
if traits.scenarioname:
startup_args.extend(['--scenario-name', traits.scenarioname])
if traits.appargs:
startup_args.extend(['--app-args', traits.appargs])
if traits.environmentvariables:
startup_args.extend(
['--environment-variables', traits.environmentvariables])
if traits.iterationsetup:
startup_args.extend(['--iteration-setup', traits.iterationsetup])
if traits.setupargs:
startup_args.extend(['--setup-args', traits.setupargs])
if traits.iterationcleanup:
startup_args.extend(
['--iteration-cleanup', traits.iterationcleanup])
if traits.cleanupargs:
startup_args.extend(['--cleanup-args', traits.cleanupargs])
if traits.measurementdelay:
startup_args.extend(
['--measurement-delay', traits.measurementdelay])
if traits.skipprofile:
startup_args.extend(['--skip-profile-iteration'])
if traits.innerloopcommand:
startup_args.extend(
['--inner-loop-command', traits.innerloopcommand])
if traits.innerloopcommandargs:
startup_args.extend(
['--inner-loop-command-args', traits.innerloopcommandargs])
if traits.runwithoutexit:
startup_args.extend(['--run-without-exit', traits.runwithoutexit])
if traits.hotreloaditers:
startup_args.extend(['--hot-reload-iters', traits.hotreloaditers])
if traits.skipmeasurementiteration:
startup_args.extend([
'--skip-measurement-iteration', traits.skipmeasurementiteration
])
upload_container = UPLOAD_CONTAINER
try:
RunCommand(startup_args, verbose=True).run()
except CalledProcessError:
getLogger().info("Run failure registered")
if runninginlab():
upload_container = 'failedresults'
reportdir = os.path.join(TRACEDIR, 'FailureReporter')
os.makedirs(reportdir)
reportjson = os.path.join(os.getcwd(), reportdir,
'failure-report.json')
cmdline = ["FailureReporting%s" % extension(), reportjson]
reporterpath = os.path.join(helixpayload(), 'FailureReporter')
if not os.path.exists(reporterpath):
raise FileNotFoundError
getLogger().info("Generating failure results at " + reportjson)
RunCommand(cmdline, verbose=True).run(reporterpath)
if runninginlab():
copytree(TRACEDIR, os.path.join(helixuploaddir(), 'traces'))
if uploadtokenpresent():
import upload
upload.upload(reportjson, upload_container, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
def __main(args: list) -> int:
validate_supported_runtime()
args = __process_arguments(args)
verbose = not args.quiet
setup_loggers(verbose=verbose)
if not args.frameworks:
raise Exception("Framework version (-f) must be specified.")
target_framework_monikers = dotnet \
.FrameworkAction \
.get_target_framework_monikers(args.frameworks)
# Acquire necessary tools (dotnet)
init_tools(architecture=args.architecture,
dotnet_versions=args.dotnet_versions,
target_framework_monikers=target_framework_monikers,
verbose=verbose)
# WORKAROUND
# The MicroBenchmarks.csproj targets .NET Core 2.1, 3.0, 3.1 and 5.0
# to avoid a build failure when using older frameworks (error NETSDK1045:
# The current .NET SDK does not support targeting .NET Core $XYZ)
# we set the TFM to what the user has provided.
os.environ['PERFLAB_TARGET_FRAMEWORKS'] = ';'.join(
target_framework_monikers)
# dotnet --info
dotnet.info(verbose=verbose)
BENCHMARKS_CSPROJ = dotnet.CSharpProject(project=args.csprojfile,
bin_directory=args.bin_directory)
if not args.run_only:
# .NET micro-benchmarks
# Restore and build micro-benchmarks
micro_benchmarks.build(BENCHMARKS_CSPROJ, args.configuration,
target_framework_monikers, args.incremental,
verbose)
# Run micro-benchmarks
if not args.build_only:
upload_container = UPLOAD_CONTAINER
try:
for framework in args.frameworks:
micro_benchmarks.run(BENCHMARKS_CSPROJ, args.configuration,
framework, verbose, args)
globpath = os.path.join(
get_artifacts_directory() if not args.bdn_artifacts else
args.bdn_artifacts, '**', '*perf-lab-report.json')
except CalledProcessError:
getLogger().info("Run failure registered")
if runninginlab():
upload_container = 'failedresults'
reportdir = os.path.join(
get_artifacts_directory() if not args.bdn_artifacts else
args.bdn_artifacts, 'FailureReporter')
os.makedirs(reportdir)
globpath = os.path.join(reportdir, 'failure-report.json')
cmdline = ["FailureReporting%s" % extension(), globpath]
reporterpath = os.path.join(helixpayload(), 'FailureReporter')
if not os.path.exists(reporterpath):
raise FileNotFoundError
getLogger().info("Generating failure results at " + globpath)
RunCommand(cmdline, verbose=True).run(reporterpath)
else:
args.upload_to_perflab_container = False
dotnet.shutdown_server(verbose)
if args.upload_to_perflab_container:
import upload
upload.upload(globpath, upload_container, UPLOAD_QUEUE,
UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
