def Build(self):
BuildType = self.env.GetValue("TARGET")
edk2_logging.log_progress("Running Build %s" % BuildType)
# set target, arch, toolchain, threads, and platform
params = "-p " + self.env.GetValue("ACTIVE_PLATFORM")
params += " -b " + BuildType
params += " -t " + self.env.GetValue("TOOL_CHAIN_TAG")
# Thread number is now optional and not set in default tianocore target.txt
if self.env.GetValue("MAX_CONCURRENT_THREAD_NUMBER") is not None:
params += " -n " + self.env.GetValue(
"MAX_CONCURRENT_THREAD_NUMBER")
# Set the arch flags. Multiple are split by space
rt = self.env.GetValue("TARGET_ARCH").split(" ")
for t in rt:
params += " -a " + t
# get the report options and setup the build command
if (self.env.GetValue("BUILDREPORTING") == "TRUE"):
params += " -y " + self.env.GetValue("BUILDREPORT_FILE")
rt = self.env.GetValue("BUILDREPORT_TYPES").split(" ")
for t in rt:
params += " -Y " + t
# add special processing to handle building a single module
mod = self.env.GetValue("BUILDMODULE")
if (mod is not None and len(mod.strip()) > 0):
params += " -m " + mod
edk2_logging.log_progress("Single Module Build: " + mod)
self.SkipPostBuild = True
self.FlashImage = False
# attach the generic build vars
buildvars = self.env.GetAllBuildKeyValues(BuildType)
for key, value in buildvars.items():
params += " -D " + key + "=" + value
output_stream = edk2_logging.create_output_stream()
env = shell_environment.ShellEnvironment()
# WORKAROUND - Pin the PYTHONHASHSEED so that TianoCore build tools
# have consistent ordering. Addresses incremental builds.
pre_build_env_chk = env.checkpoint()
env.set_shell_var('PYTHONHASHSEED', '0')
env.log_environment()
ret = RunCmd("build", params)
# WORKAROUND - Undo the workaround.
env.restore_checkpoint(pre_build_env_chk)
problems = edk2_logging.scan_compiler_output(output_stream)
edk2_logging.remove_output_stream(output_stream)
for level, problem in problems:
logging.log(level, problem)
if (ret != 0):
return ret
return 0
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
# SET PACKAGE PATH
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# Check Dependencies for Repo
for dependency in self.PlatformSettings.GetDependencies():
pplist.append(dependency["Path"])
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error(
"You need to run stewart_ci_setup to resolve all repos.")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(
f"Package Path: {self.PlatformSettings.GetPackagesPath()}")
# logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version))
# logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
# logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Enviroment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
archSupported = " ".join(self.PlatformSettings.GetArchSupported())
env.SetValue("TARGET_ARCH", archSupported,
"from PlatformSettings.GetArchSupported()")
_targets = " ".join(self.PlatformSettings.GetTargetsSupported())
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
if len(self.packageList) == 0:
self.packageList.extend(self.PlatformSettings.GetPackages())
for pkgToRunOn in self.packageList:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
ts = JunitReport.create_new_testsuite(
pkgToRunOn,
f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}"
)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txthandle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, mdhandle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txthandle, mdhandle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .mu.json
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# check the resulting configuration
config_validator.check_package_confg(pkgToRunOn, pkg_config,
pluginList)
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
for Descriptor in pluginList:
# Get our targets
targets = ["DEBUG"]
if Descriptor.Obj.IsTargetDependent() and _targets:
targets = self.PlatformSettings.GetTargetsSupported()
for target in targets:
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
pkgToRunOn, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# perhaps we should ask the validator to run on the package for this target
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc != 0):
failure_num += 1
if (rc is None):
logging.error(
"--->Test Failed: %s returned NoneType" %
Descriptor.Name)
else:
logging.error(
"--->Test Failed: %s returned %d" %
(Descriptor.Name, rc))
else:
edk2_logging.log_progress(
f"--->Test Success {Descriptor.Name} {target}")
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
Edk2CiBuild.collect_python_pip_info()
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error("Src Tree is invalid. Did you Setup correctly?")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(f"Package Path: {pplist}")
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Environment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls.
# This would be better in an edk2 plugin so that it could be modified/controlled
# more easily
#
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list),
"from edk2 ci build.py")
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
for pkgToRunOn in self.requested_package_list:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}"
ts = JunitReport.create_new_testsuite(pkgToRunOn,
package_class_name)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txt_handle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, md_handle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txt_handle, md_handle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .ci.yaml
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
# For each plugin
for Descriptor in pluginList:
# For each target
for target in self.requested_target_list:
if (target not in Descriptor.Obj.RunsOnTargetList()):
continue
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
package_class_name, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = self.merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/package path relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc > 0):
failure_num += 1
if (rc is None):
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned NoneType"
)
else:
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned {rc}"
)
elif (rc < 0):
logging.warn(
f"--->Test Skipped: in plugin! {Descriptor.Name} {target}"
)
else:
edk2_logging.log_progress(
f"--->Test Success: {Descriptor.Name} {target}"
)
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num