def RetrieveCommandLineOptions(self, args):
''' Retrieve command line options from the argparser '''
shell_environment.GetBuildVars().SetValue(
"TARGET_ARCH", args.build_arch.upper(), "From CmdLine")
shell_environment.GetBuildVars().SetValue(
"ACTIVE_PLATFORM", os.path.join("VExpressPkg", "ArmVExpress-FVP-AArch64.dsc"), "From CmdLine")
def RetrieveCommandLineOptions(self, args):
''' Retrieve command line options from the argparser '''
shell_environment.GetBuildVars().SetValue("TARGET_ARCH",
args.build_arch.upper(),
"From CmdLine")
shell_environment.GetBuildVars().SetValue(
"ACTIVE_PLATFORM", "EmulatorPkg/EmulatorPkg.dsc", "From CmdLine")
def Go(self):
ret = 0
env = shell_environment.GetBuildVars()
env.SetValue("PRODUCT_NAME", self.PlatformSettings.GetName(),
"Platform Hardcoded")
env.SetValue("BLD_*_BUILDID_STRING", "201905", "Current Version")
env.SetValue("BUILDREPORTING", "TRUE", "Platform Hardcoded")
env.SetValue("BUILDREPORT_TYPES", 'PCD DEPEX LIBRARY BUILD_FLAGS',
"Platform Hardcoded")
# make sure python_command is set
python_command = sys.executable
if " " in python_command:
python_command = '"' + python_command + '"'
shell_environment.GetEnvironment().set_shell_var(
"PYTHON_COMMAND", python_command)
# Run pre build hook
ret += self.PlatformSettings.PreFirstBuildHook()
ws = self.GetWorkspaceRoot()
pp = self.PlatformSettings.GetModulePkgsPath()
# run each configuration
ret = 0
try:
for config in self.PlatformSettings.GetConfigurations():
pre_ret = self.PlatformSettings.PreBuildHook(
) # run pre build hook
if pre_ret != 0:
ret = pre_ret
raise RuntimeError("We failed in prebuild hook")
edk2_logging.log_progress(f"--Running next configuration--")
logging.info(config)
shell_environment.CheckpointBuildVars(
) # checkpoint our config
env = shell_environment.GetBuildVars()
# go through the config and apply to environement
for key in config:
env.SetValue(key, config[key], "provided by configuration")
# make sure to set this after in case the config did
env.SetValue("TOOL_CHAIN_TAG", "VS2017", "provided by builder")
platformBuilder = UefiBuilder() # create our builder
build_ret = platformBuilder.Go(ws, pp, self.helper,
self.plugin_manager)
# we always want to run the post build hook
post_ret = self.PlatformSettings.PostBuildHook(ret)
if build_ret != 0:
ret = build_ret
raise RuntimeError("We failed in build")
if post_ret != 0:
ret = post_ret
raise RuntimeError("We failed in postbuild hook")
shell_environment.RevertBuildVars()
except RuntimeError:
pass
finally:
# make sure to do our final build hook
self.PlatformSettings.PostFinalBuildHook(ret)
return ret
def RetrieveCommandLineOptions(self, args):
''' Retrieve command line options from the argparser '''
shell_environment.GetBuildVars().SetValue(
"TARGET_ARCH", " ".join(args.build_arch.upper().split(",")),
"From CmdLine")
dsc = CommonPlatform.GetDscName(args.build_arch)
shell_environment.GetBuildVars().SetValue("ACTIVE_PLATFORM",
f"OvmfPkg/{dsc}",
"From CmdLine")
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
scopes = CommonPlatform.Scopes
ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
Arch = shell_environment.GetBuildVars().GetValue("TARGET_ARCH", "")
if GetHostInfo().os.upper() == "LINUX" and ActualToolChainTag.upper().startswith("GCC"):
if "AARCH64" == Arch:
scopes += ("gcc_aarch64_linux",)
elif "ARM" == Arch:
scopes += ("gcc_arm_linux",)
return scopes
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
if self.ActualScopes is None:
scopes = ("cibuild", "edk2-build", "host-based-test")
self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
is_linux = GetHostInfo().os.upper() == "LINUX"
if self.UseBuiltInBaseTools is None:
is_linux = GetHostInfo().os.upper() == "LINUX"
# try and import the pip module for basetools
try:
import edk2basetools
self.UseBuiltInBaseTools = True
except ImportError:
self.UseBuiltInBaseTools = False
pass
if self.UseBuiltInBaseTools == True:
scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
logging.warning("Using Pip Tools based BaseTools")
else:
logging.warning("Falling back to using in-tree BaseTools")
if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):
if "AARCH64" in self.ActualArchitectures:
scopes += ("gcc_aarch64_linux",)
if "ARM" in self.ActualArchitectures:
scopes += ("gcc_arm_linux",)
if "RISCV64" in self.ActualArchitectures:
scopes += ("gcc_riscv64_unknown",)
self.ActualScopes = scopes
return self.ActualScopes
def test_special_build_vars_should_be_checkpointable(self):
shell_env = SE.ShellEnvironment()
build_vars = SE.GetBuildVars()
# This test is basically a rehash of the object checkpointing test,
# but this time with the special vars.
test_var1_name = 'SE_TEST_VAR_3'
test_var1_data = 'MyData1'
test_var1_data2 = 'RevisedData1'
test_var1_data3 = 'MoreRevisedData1'
test_var2_name = 'SE_TEST_VAR_4'
test_var2_data = 'MyData2'
# Set the first data and make a checkpoint.
build_vars.SetValue(test_var1_name,
test_var1_data,
'var1 set',
overridable=True)
check_point1 = shell_env.checkpoint()
# Update previous value and set second data. Then checkpoint.
build_vars.SetValue(test_var1_name,
test_var1_data2,
'var1 set',
overridable=True)
build_vars.SetValue(test_var2_name,
test_var2_data,
'var2 set',
overridable=True)
check_point2 = shell_env.checkpoint()
# Restore the first checkpoint and verify values.
shell_env.restore_checkpoint(check_point1)
self.assertEqual(shell_env.get_build_var(test_var1_name),
test_var1_data)
self.assertIs(shell_env.get_build_var(test_var2_name), None)
# Make a change to be tested later.
build_vars.SetValue(test_var1_name,
test_var1_data3,
'var1 set',
overridable=True)
self.assertEqual(
shell_env.get_build_var(test_var1_name), test_var1_data3,
'even after restore, special build vars should always update current'
)
# Restore the second checkpoint and verify values.
shell_env.restore_checkpoint(check_point2)
self.assertEqual(shell_env.get_build_var(test_var1_name),
test_var1_data2)
self.assertEqual(shell_env.get_build_var(test_var2_name),
test_var2_data)
# Restore the first checkpoint again and make sure original value still stands.
shell_env.restore_checkpoint(check_point1)
self.assertEqual(shell_env.get_build_var(test_var1_name),
test_var1_data)
def test_special_build_vars_should_always_update_current(self):
shell_env = SE.ShellEnvironment()
build_vars = SE.GetBuildVars()
test_var1_name = 'SE_TEST_VAR_update_current1'
test_var1_data = 'NewData1'
test_var1_data2 = 'NewerData1'
test_var2_name = 'SE_TEST_VAR_update_current2'
test_var2_data = 'NewData2'
# Make a change and checkpoint.
build_vars.SetValue(test_var1_name,
test_var1_data,
'var1 set',
overridable=True)
shell_env.checkpoint()
# Make a couple more changes.
build_vars.SetValue(test_var1_name,
test_var1_data2,
'var1 set',
overridable=True)
build_vars.SetValue(test_var2_name,
test_var2_data,
'var2 set',
overridable=True)
# Make sure that the newer changes are valid.
self.assertEqual(shell_env.get_build_var(test_var1_name),
test_var1_data2)
self.assertEqual(shell_env.get_build_var(test_var2_name),
test_var2_data)
def GetPackagesPath(self):
''' Return a list of workspace relative paths that should be mapped as edk2 PackagesPath '''
result = [
shell_environment.GetBuildVars().GetValue("FEATURE_CONFIG_PATH", "")
]
for a in CommonPlatform.PackagesPath:
result.append(a)
return result
def __init__(self, OverrideConf, AdditionalTemplateConfDir):
self.Logger = logging.getLogger("ConfMgmt")
self.env = shell_environment.GetBuildVars()
if (self.env.GetValue("WORKSPACE") is None) or \
(self.env.GetValue("EDK2_BASE_TOOLS_DIR") is None):
raise Exception(
"WORKSPACE and EDK2_BASE_TOOLS_DIR must be set prior to running ConfMgmt"
)
self.__PopulateConf(OverrideConf, AdditionalTemplateConfDir)
def GetName(self):
''' Get the name of the repo, platform, or product being build '''
''' Used for naming the log file, among others '''
# check the startup nsh flag and if set then rename the log file.
# this helps in CI so we don't overwrite the build log since running
# uses the stuart_build command.
if(shell_environment.GetBuildVars().GetValue("MAKE_STARTUP_NSH", "FALSE") == "TRUE"):
return "OvmfPkg_With_Run"
return "OvmfPkg"
def test_get_build_vars_should_update_vars(self):
shell_env = SE.ShellEnvironment()
build_vars = SE.GetBuildVars()
test_var_name = 'SE_TEST_VAR_4'
test_var_data = 'NewData1'
build_vars.SetValue(test_var_name, test_var_data, 'random set')
self.assertEqual(shell_env.get_build_var(test_var_name), test_var_data)
def RetrieveCommandLineOptions(self, args):
''' Retrieve command line options from the argparser '''
shell_environment.GetBuildVars().SetValue("TOOL_CHAIN_TAG", "VS2017",
"Set default")
if args.api_key is not None:
self.api_key = args.api_key
print("Using API KEY")
self.nuget_version = args.nug_ver
self.should_dump_version = args.dump_version
def test_go_skip_building(self):
builder = uefi_build.UefiBuilder()
builder.SkipPostBuild = True
builder.SkipBuild = True
builder.SkipBuild = True
manager = PluginManager()
shell_environment.GetBuildVars().SetValue("EDK_TOOLS_PATH", self.WORKSPACE, "empty")
helper = uefi_helper_plugin.HelperFunctions()
ret = builder.Go(self.WORKSPACE, "", helper, manager)
self.assertEqual(ret, 0)
def GetPackagesPath(self):
''' Return a list of workspace relative paths that should be mapped as edk2-platforms PackagesPath '''
packages = []
edk2_repo = shell_environment.GetBuildVars().GetValue("EDK2_REPO", "")
if not edk2_repo:
edk2_repo = os.path.join("edk2_extdep", "edk2")
packages.append(edk2_repo)
packages.append(os.path.join("Platform", "ARM"))
return packages
def test_special_build_vars_should_default_non_overrideable(self):
shell_env = SE.ShellEnvironment()
build_vars = SE.GetBuildVars()
test_var_name = 'SE_TEST_VAR_4'
test_var_data = 'NewData1'
test_var_data2 = 'NewerData1'
build_vars.SetValue(test_var_name, test_var_data, 'random set')
build_vars.SetValue(test_var_name, test_var_data2, 'another random set')
self.assertEqual(shell_env.get_build_var(test_var_name), test_var_data)
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
scopes = ("cibuild","edk2-build")
self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
if GetHostInfo().os.upper() == "LINUX" and self.ActualToolChainTag.upper().startswith("GCC"):
if "AARCH64" in self.ActualArchitectures:
scopes += ("gcc_aarch64_linux",)
if "ARM" in self.ActualArchitectures:
scopes += ("gcc_arm_linux",)
return scopes
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
scopes = ("corebuild", "project_mu","cibuild")
self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
if (GetHostInfo().os == "Linux" and "AARCH64" in self.ActualArchitectures and self.ActualToolChainTag.upper().startswith("GCC")):
scopes += ("gcc_aarch64_linux",)
if GetHostInfo().os == "Windows":
scopes += ("host-test-win",)
return scopes
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
if self.ActualScopes is None:
scopes = ("cibuild", "edk2-build", "host-based-test")
self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
is_linux = GetHostInfo().os.upper() == "LINUX"
scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):
if "AARCH64" in self.ActualArchitectures:
scopes += ("gcc_aarch64_linux",)
if "ARM" in self.ActualArchitectures:
scopes += ("gcc_arm_linux",)
if "RISCV64" in self.ActualArchitectures:
scopes += ("gcc_riscv64_unknown",)
# If EDK2_REPO is not provided, download it.
if not shell_environment.GetBuildVars().GetValue("EDK2_REPO", ""):
scopes += ("edk2-repo",)
self.ActualScopes = scopes
return self.ActualScopes
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
# SET PACKAGE PATH
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# Check Dependencies for Repo
for dependency in self.PlatformSettings.GetDependencies():
pplist.append(dependency["Path"])
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error(
"You need to run stewart_ci_setup to resolve all repos.")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(
f"Package Path: {self.PlatformSettings.GetPackagesPath()}")
# logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version))
# logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
# logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Enviroment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
archSupported = " ".join(self.PlatformSettings.GetArchSupported())
env.SetValue("TARGET_ARCH", archSupported,
"from PlatformSettings.GetArchSupported()")
_targets = " ".join(self.PlatformSettings.GetTargetsSupported())
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
if len(self.packageList) == 0:
self.packageList.extend(self.PlatformSettings.GetPackages())
for pkgToRunOn in self.packageList:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
ts = JunitReport.create_new_testsuite(
pkgToRunOn,
f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}"
)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txthandle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, mdhandle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txthandle, mdhandle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .mu.json
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# check the resulting configuration
config_validator.check_package_confg(pkgToRunOn, pkg_config,
pluginList)
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
for Descriptor in pluginList:
# Get our targets
targets = ["DEBUG"]
if Descriptor.Obj.IsTargetDependent() and _targets:
targets = self.PlatformSettings.GetTargetsSupported()
for target in targets:
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
pkgToRunOn, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# perhaps we should ask the validator to run on the package for this target
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc != 0):
failure_num += 1
if (rc is None):
logging.error(
"--->Test Failed: %s returned NoneType" %
Descriptor.Name)
else:
logging.error(
"--->Test Failed: %s returned %d" %
(Descriptor.Name, rc))
else:
edk2_logging.log_progress(
f"--->Test Success {Descriptor.Name} {target}")
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num
def RetrieveCommandLineOptions(self, args):
if args.tool_chain_tag is not None:
shell_environment.GetBuildVars().SetValue("TOOL_CHAIN_TAG", args.tool_chain_tag, "Set as cli parameter")
# cache this so usage within CISettings is consistant.
self.ToolChainTagCacheValue = args.tool_chain_tag
def SetEnv(self):
edk2_logging.log_progress("Setting up the Environment")
shell_environment.GetEnvironment().set_shell_var("WORKSPACE", self.ws)
shell_environment.GetBuildVars().SetValue("WORKSPACE", self.ws,
"Set in SetEnv")
if (self.pp is not None):
shell_environment.GetEnvironment().set_shell_var(
"PACKAGES_PATH", self.pp)
shell_environment.GetBuildVars().SetValue("PACKAGES_PATH", self.pp,
"Set in SetEnv")
# process platform parameters defined in platform build file
ret = self.SetPlatformEnv()
if (ret != 0):
logging.critical("Set Platform Env failed")
return ret
# set some basic defaults
self.SetBasicDefaults()
# Handle all the template files for workspace/conf/ Allow override
TemplateDirList = [self.env.GetValue("EDK_TOOLS_PATH")
] # set to edk2 BaseTools
PlatTemplatesForConf = self.env.GetValue(
"CONF_TEMPLATE_DIR") # get platform defined additional path
if (PlatTemplatesForConf is not None):
PlatTemplatesForConf = self.mws.join(self.ws, PlatTemplatesForConf)
TemplateDirList.insert(0, PlatTemplatesForConf)
logging.debug(
f"Platform defined override for Template Conf Files: {PlatTemplatesForConf}"
)
conf_dir = os.path.join(self.ws, "Conf")
conf_mgmt.ConfMgmt().populate_conf_dir(conf_dir, self.UpdateConf,
TemplateDirList)
# parse target file
ret = self.ParseTargetFile()
if (ret != 0):
logging.critical("ParseTargetFile failed")
return ret
# parse DSC file
ret = self.ParseDscFile()
if (ret != 0):
logging.critical("ParseDscFile failed")
return ret
# parse FDF file
ret = self.ParseFdfFile()
if (ret != 0):
logging.critical("ParseFdfFile failed")
return ret
# set build output base envs for all builds
if self.env.GetValue("OUTPUT_DIRECTORY") is None:
logging.warn("OUTPUT_DIRECTORY was not found, defaulting to Build")
self.env.SetValue("OUTPUT_DIRECTORY", "Build",
"default from uefi_build", True)
self.env.SetValue(
"BUILD_OUT_TEMP",
os.path.join(self.ws, self.env.GetValue("OUTPUT_DIRECTORY")),
"Computed in SetEnv")
target = self.env.GetValue("TARGET")
self.env.SetValue(
"BUILD_OUTPUT_BASE",
os.path.join(self.env.GetValue("BUILD_OUT_TEMP"),
target + "_" + self.env.GetValue("TOOL_CHAIN_TAG")),
"Computed in SetEnv")
# We have our build target now. Give platform build one more chance for target specific settings.
ret = self.SetPlatformEnvAfterTarget()
if (ret != 0):
logging.critical("SetPlatformEnvAfterTarget failed")
return ret
# set the build report file
self.env.SetValue(
"BUILDREPORT_FILE",
os.path.join(self.env.GetValue("BUILD_OUTPUT_BASE"),
"BUILD_REPORT.TXT"), True)
# set environment variables for the build process
os.environ["EFI_SOURCE"] = self.ws
return 0
def ParseCommandLineOptions(self):
'''
Parses command line options.
Sets up argparser specifically to get PlatformSettingsManager instance.
Then sets up second argparser and passes it to child class and to PlatformSettingsManager.
Finally, parses all known args and then reads the unknown args in to build vars.
'''
# first argparser will only get settings manager and help will be disabled
settingsParserObj = argparse.ArgumentParser(add_help=False)
# instantiate the second argparser that will get passed around
epilog = '''
<key>=<value> - Set an env variable for the pre/post build process
BLD_*_<key>=<value> - Set a build flag for all build types.
Key=value will get passed to build process
BLD_<TARGET>_<key>=<value> - Set a build flag for build type of <target>
Key=value will get passed to build process for given build type)'''
parserObj = argparse.ArgumentParser(epilog=epilog)
settingsParserObj.add_argument(
'-c',
'--platform_module',
dest='platform_module',
default="PlatformBuild.py",
type=str,
help=
'Provide the Platform Module relative to the current working directory.'
f'This should contain a {self.GetSettingsClass().__name__} instance.'
)
# get the settings manager from the provided file and load an instance
settingsArg, unknown_args = settingsParserObj.parse_known_args()
try:
self.PlatformModule = import_module_by_file_name(
os.path.abspath(settingsArg.platform_module))
self.PlatformSettings = locate_class_in_module(
self.PlatformModule, self.GetSettingsClass())()
except (TypeError):
# Gracefully exit if the file we loaded isn't the right type
class_name = self.GetSettingsClass().__name__
print(
f"Unable to use {settingsArg.platform_module} as a {class_name}"
)
print("Did you mean to use a different kind of invocable?")
try:
# If this works, we can provide help for whatever special functions
# the subclass is offering.
self.AddCommandLineOptions(settingsParserObj)
Module = self.PlatformModule
module_contents = dir(Module)
# Filter through the Module, we're only looking for classes.
classList = [
getattr(Module, obj) for obj in module_contents
if inspect.isclass(getattr(Module, obj))
]
# Get the name of all the classes
classNameList = [obj.__name__ for obj in classList]
# TODO improve filter to no longer catch imports as well as declared classes
imported_classes = ", ".join(
classNameList) # Join the classes together
print(f"The module you imported contains {imported_classes}")
except Exception:
# Otherwise, oh well we'll just ignore this.
raise
pass
settingsParserObj.print_help()
sys.exit(1)
except (FileNotFoundError):
# Gracefully exit if we can't find the file
print(f"We weren't able to find {settingsArg.platform_module}")
settingsParserObj.print_help()
sys.exit(2)
except Exception as e:
print(
f"Error: We had trouble loading {settingsArg.platform_module}. Is the path correct?"
)
# Gracefully exit if setup doesn't go well.
settingsParserObj.print_help()
print(e)
sys.exit(2)
# now to get the big arg parser going...
# first pass it to the subclass
self.AddCommandLineOptions(parserObj)
# next pass it to the settings manager
self.PlatformSettings.AddCommandLineOptions(parserObj)
default_build_config_path = os.path.join(self.GetWorkspaceRoot(),
"BuildConfig.conf")
# add the common stuff that everyone will need
parserObj.add_argument('--build-config',
dest='build_config',
default=default_build_config_path,
type=str,
help='Provide shell variables in a file')
parserObj.add_argument('--verbose',
'--VERBOSE',
'-v',
dest="verbose",
action='store_true',
default=False,
help='verbose')
# setup sys.argv and argparse round 2
sys.argv = [sys.argv[0]] + unknown_args
args, unknown_args = parserObj.parse_known_args()
self.Verbose = args.verbose
# give the parsed args to the subclass
self.RetrieveCommandLineOptions(args)
# give the parsed args to platform settings manager
self.PlatformSettings.RetrieveCommandLineOptions(args)
#
# Look through unknown_args and BuildConfig for strings that are x=y,
# set env.SetValue(x, y),
# then remove this item from the list.
#
env = shell_environment.GetBuildVars()
BuildConfig = os.path.abspath(args.build_config)
for argument in unknown_args:
if argument.count("=") != 1:
raise RuntimeError(
f"Unknown variable passed in via CLI: {argument}")
tokens = argument.strip().split("=")
env.SetValue(tokens[0].strip().upper(), tokens[1].strip(),
"From CmdLine")
unknown_args.clear() # remove the arguments we've already consumed
if os.path.isfile(BuildConfig):
with open(BuildConfig) as file:
for line in file:
stripped_line = line.strip().partition("#")[0]
if len(stripped_line) == 0:
continue
unknown_args.append(stripped_line)
for argument in unknown_args:
if argument.count("=") != 1:
raise RuntimeError(
f"Unknown variable passed in via BuildConfig: {argument}")
tokens = argument.strip().split("=")
env.SetValue(tokens[0].strip().upper(), tokens[1].strip(),
"From BuildConf")
def __init__(self):
plugin_skip_list = ["DependencyCheck"]
env = shell_environment.GetBuildVars()
for plugin in plugin_skip_list:
env.SetValue(plugin.upper(), "skip", "set from settings file")
pass
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
Edk2CiBuild.collect_python_pip_info()
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error("Src Tree is invalid. Did you Setup correctly?")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(f"Package Path: {pplist}")
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Environment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls.
# This would be better in an edk2 plugin so that it could be modified/controlled
# more easily
#
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list),
"from edk2 ci build.py")
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
for pkgToRunOn in self.requested_package_list:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}"
ts = JunitReport.create_new_testsuite(pkgToRunOn,
package_class_name)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txt_handle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, md_handle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txt_handle, md_handle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .ci.yaml
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
# For each plugin
for Descriptor in pluginList:
# For each target
for target in self.requested_target_list:
if (target not in Descriptor.Obj.RunsOnTargetList()):
continue
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
package_class_name, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = self.merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/package path relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc > 0):
failure_num += 1
if (rc is None):
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned NoneType"
)
else:
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned {rc}"
)
elif (rc < 0):
logging.warn(
f"--->Test Skipped: in plugin! {Descriptor.Name} {target}"
)
else:
edk2_logging.log_progress(
f"--->Test Success: {Descriptor.Name} {target}"
)
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num
def SetEnv(self):
edk2_logging.log_progress("Setting up the Environment")
shell_environment.GetEnvironment().set_shell_var("WORKSPACE", self.ws)
shell_environment.GetBuildVars().SetValue("WORKSPACE", self.ws,
"Set in SetEnv")
if (self.pp is not None):
shell_environment.GetEnvironment().set_shell_var(
"PACKAGES_PATH", self.pp)
shell_environment.GetBuildVars().SetValue("PACKAGES_PATH", self.pp,
"Set in SetEnv")
# process platform parameters defined in platform build file
ret = self.SetPlatformEnv()
if (ret != 0):
logging.critical("Set Platform Env failed")
return ret
# set some basic defaults
self.SetBasicDefaults()
# Handle all the template files for workspace/conf/ Allow override
TemplatesForConf = self.env.GetValue("CONF_TEMPLATE_DIR")
if (TemplatesForConf is not None):
TemplatesForConf = self.mws.join(self.ws, TemplatesForConf)
logging.debug(
"Platform defined override for Template Conf Files: %s",
TemplatesForConf)
e = conf_mgmt.ConfMgmt(self.UpdateConf, TemplatesForConf)
# parse target file
ret = self.ParseTargetFile()
if (ret != 0):
logging.critical("ParseTargetFile failed")
return ret
ret = e.ToolsDefConfigure()
if (ret != 0):
logging.critical("ParseTargetFile failed")
return ret
# parse DSC file
ret = self.ParseDscFile()
if (ret != 0):
logging.critical("ParseDscFile failed")
return ret
# parse FDF file
ret = self.ParseFdfFile()
if (ret != 0):
logging.critical("ParseFdfFile failed")
return ret
# set build output base envs for all builds
self.env.SetValue(
"BUILD_OUT_TEMP",
os.path.join(self.ws, self.env.GetValue("OUTPUT_DIRECTORY")),
"Computed in SetEnv")
target = self.env.GetValue("TARGET")
self.env.SetValue(
"BUILD_OUTPUT_BASE",
os.path.join(self.env.GetValue("BUILD_OUT_TEMP"),
target + "_" + self.env.GetValue("TOOL_CHAIN_TAG")),
"Computed in SetEnv")
# We have our build target now. Give platform build one more chance for target specific settings.
ret = self.SetPlatformEnvAfterTarget()
if (ret != 0):
logging.critical("SetPlatformEnvAfterTarget failed")
return ret
# set the build report file
self.env.SetValue(
"BUILDREPORT_FILE",
os.path.join(self.env.GetValue("BUILD_OUTPUT_BASE"),
"BUILD_REPORT.TXT"), True)
# set environment variables for the build process
os.environ["EFI_SOURCE"] = self.ws
return 0
def ParseCommandLineOptions(self):
'''
Parses command line options.
Sets up argparser specifically to get PlatformSettingsManager instance.
Then sets up second argparser and passes it to child class and to PlatformSettingsManager.
Finally, parses all known args and then reads the unknown args in to build vars.
'''
# first argparser will only get settings manager and help will be disabled
settingsParserObj = argparse.ArgumentParser(add_help=False)
# instantiate the second argparser that will get passed around
epilog = '''
<key>=<value> - Set an env variable for the pre/post build process
BLD_*_<key>=<value> - Set a build flag for all build types.
Key=value will get passed to build process
BLD_<TARGET>_<key>=<value> - Set a build flag for build type of <target>
Key=value will get passed to build process for given build type)'''
parserObj = argparse.ArgumentParser(epilog=epilog)
settingsParserObj.add_argument(
'-c',
'--platform_module',
dest='platform_module',
default="PlatformBuild.py",
type=str,
help=
'Provide the Platform Module relative to the current working directory.'
f'This should contain a {self.GetSettingsClass().__name__} instance.'
)
# get the settings manager from the provided file and load an instance
settingsArg, unknown_args = settingsParserObj.parse_known_args()
try:
self.PlatformModule = import_module_by_file_name(
os.path.abspath(settingsArg.platform_module))
self.PlatformSettings = locate_class_in_module(
self.PlatformModule, self.GetSettingsClass())()
except (TypeError, FileNotFoundError) as e:
# Gracefully exit if setup doesn't go well.
try:
# If this works, we can provide help for whatever special functions
# the subclass is offering.
self.AddCommandLineOptions(settingsParserObj)
except:
# If it didn't work, oh well.
pass
print(e)
settingsParserObj.print_help()
sys.exit(0)
# now to get the big arg parser going...
# first pass it to the subclass
self.AddCommandLineOptions(parserObj)
# next pass it to the settings manager
self.PlatformSettings.AddCommandLineOptions(parserObj)
default_build_config_path = os.path.join(self.GetWorkspaceRoot(),
"BuildConfig.conf")
# add the common stuff that everyone will need
parserObj.add_argument('--build-config',
dest='build_config',
default=default_build_config_path,
type=str,
help='Provide shell variables in a file')
parserObj.add_argument('--verbose',
'--VERBOSE',
'-v',
dest="verbose",
action='store_true',
default=False,
help='verbose')
# setup sys.argv and argparse round 2
sys.argv = [sys.argv[0]] + unknown_args
args, unknown_args = parserObj.parse_known_args()
self.Verbose = args.verbose
# give the parsed args to the subclass
self.RetrieveCommandLineOptions(args)
# give the parsed args to platform settings manager
self.PlatformSettings.RetrieveCommandLineOptions(args)
#
# Look through unknown_args and BuildConfig for strings that are x=y,
# set env.SetValue(x, y),
# then remove this item from the list.
#
env = shell_environment.GetBuildVars()
BuildConfig = os.path.abspath(args.build_config)
if os.path.isfile(BuildConfig):
with open(BuildConfig) as file:
for line in file:
stripped_line = line.strip()
if not stripped_line.startswith("#"):
unknown_args.append(line)
i = 0
while i < len(unknown_args):
unknown_arg = unknown_args[i]
if (unknown_arg.count("=") == 1):
tokens = unknown_arg.strip().split("=")
env.SetValue(tokens[0].strip().upper(), tokens[1].strip(),
"From CmdLine")
del unknown_args[i]
else:
i += 1
if len(unknown_args) > 0:
parserObj.print_help()
raise RuntimeError(f"Unknown variables passed in: {unknown_args}")
def Go(self, WorkSpace, PackagesPath, PInHelper, PInManager):
self.env = shell_environment.GetBuildVars()
self.mws = MultipleWorkspace()
self.mws.setWs(WorkSpace, PackagesPath)
self.ws = WorkSpace
self.pp = PackagesPath # string using os.pathsep
self.Helper = PInHelper
self.pm = PInManager
try:
edk2_logging.log_progress("Start time: {0}".format(
datetime.datetime.now()))
start_time = time.perf_counter()
self.Helper.DebugLogRegisteredFunctions()
ret = self.SetEnv()
if (ret != 0):
logging.critical("SetEnv failed")
return ret
# clean
if (self.Clean):
edk2_logging.log_progress("Cleaning")
ret = self.CleanTree()
if (ret != 0):
logging.critical("Clean failed")
return ret
# prebuild
if (self.SkipPreBuild):
edk2_logging.log_progress("Skipping Pre Build")
else:
ret = self.PreBuild()
if (ret != 0):
logging.critical("Pre Build failed")
return ret
# Output Build Environment to File - this is mostly for debug of build
# issues or adding other build features using existing variables
if (self.OutputConfig is not None):
edk2_logging.log_progress("Writing Build Env Info out to File")
logging.debug("Found an Output Build Env File: " +
self.OutputConfig)
self.env.PrintAll(self.OutputConfig)
if (self.env.GetValue("GATEDBUILD")
is not None) and (self.env.GetValue("GATEDBUILD").upper()
== "TRUE"):
ShouldGatedBuildRun = self.PlatformGatedBuildShouldHappen()
logging.debug("Platform Gated Build Should Run returned: %s" %
str(ShouldGatedBuildRun))
if (not self.SkipBuild):
self.SkipBuild = not ShouldGatedBuildRun
if (not self.SkipPostBuild):
self.SkipPostBuild = not ShouldGatedBuildRun
# build
if (self.SkipBuild):
edk2_logging.log_progress("Skipping Build")
else:
ret = self.Build()
if (ret != 0):
logging.critical("Build failed")
return ret
# postbuild
if (self.SkipPostBuild):
edk2_logging.log_progress("Skipping Post Build")
else:
ret = self.PostBuild()
if (ret != 0):
logging.critical("Post Build failed")
return ret
# flash
if (self.FlashImage):
edk2_logging.log_progress("Flashing Image")
ret = self.FlashRomImage()
if (ret != 0):
logging.critical("Flash Image failed")
return ret
except:
logging.critical("Build Process Exception")
logging.error(traceback.format_exc())
return -1
finally:
end_time = time.perf_counter()
elapsed_time_s = int((end_time - start_time))
edk2_logging.log_progress(
"End time: {0}\t Total time Elapsed: {1}".format(
datetime.datetime.now(),
datetime.timedelta(seconds=elapsed_time_s)))
return 0
def GetConfigurationLabel(self):
env = shell_environment.GetBuildVars()
target = env.GetValue("TARGET")
arch = env.GetValue("TARGET_ARCH")
label_parts = [target, arch]
return "_".join(label_parts)
