def test_pp_outside_workspace(self):
''' test with packagespath pointing to folder outside of workspace
root/ <-- current working directory
folder_ws/ <-- workspace root
ws packages here
folder_pp/ <-- packages path
pp packages here
'''
ws_rel = "folder_ws"
ws_abs = os.path.join(self.tmp, ws_rel)
os.mkdir(ws_abs)
folder_pp_rel = "pp1"
folder_pp1_abs = os.path.join(self.tmp, folder_pp_rel)
os.mkdir(folder_pp1_abs)
# pp absolute
pathobj = Edk2Path(ws_abs, [folder_pp1_abs])
self.assertEqual(pathobj.WorkspacePath, ws_abs)
self.assertEqual(pathobj.PackagePathList[0], folder_pp1_abs)
# pp relative to cwd
pathobj = Edk2Path(ws_abs, [folder_pp_rel])
self.assertEqual(pathobj.WorkspacePath, ws_abs)
self.assertEqual(pathobj.PackagePathList[0], folder_pp1_abs)
def test_nonexistant_abs(self):
''' test edk2path with valid ws but invalid pp'''
pp = "doesnot_exist"
pp_full_1 = os.path.join(self.tmp, pp)
# absolute path
with self.assertRaises(Exception):
Edk2Path(self.tmp, [pp_full_1])
# relative path
with self.assertRaises(Exception):
Edk2Path(self.tmp, [pp])
# confirm optional parameter to remove invalid pp values
pathobj = Edk2Path(self.tmp, [pp], error_on_invalid_pp=False)
self.assertEqual(len(pathobj.PackagePathList), 0)
def override_plat_validate(self, thebuilder):
result = self.OverrideResult.OR_ALL_GOOD
InfFileList = self.get_dsc_inf_list(thebuilder)
ws = thebuilder.ws
pp = thebuilder.pp.split(os.pathsep)
self.PathTool = Edk2Path(ws, pp)
if (InfFileList == []):
return result
modulelist = []
status = [0, 0]
# Search through the workspace and package paths
for file in InfFileList:
temp_list = []
modulenode = self.ModuleNode(file, self.OverrideResult.OR_ALL_GOOD, 0)
fullpath = thebuilder.mws.join(thebuilder.ws, file)
m_result = self.override_detect_process(thebuilder, fullpath, temp_list, modulenode, status)
# Do not log the module that does not have any override records
if (len(modulenode.reflist) > 0):
modulelist.append(modulenode)
if m_result != self.OverrideResult.OR_ALL_GOOD:
if m_result != self.OverrideResult.OR_DSC_INF_NOT_FOUND:
result = m_result
logging.error("Override processing error %s in file %s" % (self.OverrideResult.GetErrStr(m_result), file))
self.override_log_print(thebuilder, modulelist, status)
return result
def test_basic_init_ws_cwd(self):
''' test edk2path with a relative path to workspace'''
relpath = "testrootfolder"
fullpath = os.path.join(self.tmp, relpath)
os.mkdir(fullpath)
pathobj = Edk2Path(relpath, [])
self.assertEqual(pathobj.WorkspacePath, fullpath)
def test_basic_init_ws_abs_different_case(self):
inputPath = self.tmp.capitalize()
if self.tmp[0].isupper():
inputPath = self.tmp[0].lower() + self.tmp[1:]
pathobj = Edk2Path(inputPath, [])
self.assertNotEqual(pathobj.WorkspacePath, self.tmp)
def test_get_edk2_relative_path_from_absolute_path(self):
''' test basic usage of GetEdk2RelativePathFromAbsolutePath with packages path nested
inside the workspace
File layout:
root/ <-- current working directory (self.tmp)
folder_ws/ <-- workspace root
folder_pp/ <-- packages path
PPTestPkg/ <-- A edk2 package
PPTestPkg.DEC
module1/
module1.INF
module2/
module2.INF
X64/
TestFile.c
WSTestPkg/ <-- A edk2 package
WSTestPkg.dec
module1/
module1.inf
module2/
module2.inf
X64/
TestFile.c
'''
ws_rel = "folder_ws"
ws_abs = os.path.join(self.tmp, ws_rel)
os.mkdir(ws_abs)
folder_pp_rel = "pp1"
folder_pp1_abs = os.path.join(ws_abs, folder_pp_rel)
os.mkdir(folder_pp1_abs)
ws_p_name = "WSTestPkg"
ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name)
pp_p_name = "PPTestPkg"
pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False)
pathobj = Edk2Path(ws_abs, [folder_pp1_abs])
# file in packages path
p = os.path.join(pp_pkg_abs, "module1", "module1.INF")
self.assertEqual(pathobj.GetEdk2RelativePathFromAbsolutePath(p), f"{pp_p_name}/module1/module1.INF")
# file in workspace
p = os.path.join(ws_pkg_abs, "module2", "X64", "TestFile.c")
self.assertEqual(pathobj.GetEdk2RelativePathFromAbsolutePath(p), f"{ws_p_name}/module2/X64/TestFile.c")
# file not in workspace
p = os.path.join(self.tmp, "module2", "X64", "TestFile.c")
self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(p))
# pass in bad parameter
self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(None))
# file is outside of code tree and not absolute path
p = os.path.join("module2", "X64", "TestFile.c")
self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(p))
# file is cwd relative but not absolute path
p = os.path.join(ws_rel, ws_p_name, "module2", "X64", "TestFile.c")
self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(p))
def _initialize_environment_info(self, package_rel_path: str, edk2_path: Edk2Path, package_config: Dict[str, List[str]], tc: JunitReportTestCase) -> None:
"""
Initializes plugin environment information.
"""
self._abs_package_path = edk2_path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
package_rel_path)
self._abs_workspace_path = edk2_path.WorkspacePath
self._package_config = package_config
self._package_name = os.path.basename(
os.path.normpath(package_rel_path))
self._plugin_name = self.__class__.__name__
self._plugin_path = os.path.dirname(os.path.realpath(__file__))
self._rel_package_path = package_rel_path
self._tc = tc
def Go(self):
logging.info("Running Python version: " + str(sys.version_info))
Edk2PlatformBuild.collect_python_pip_info()
(build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
# Load plugins
logging.log(edk2_logging.SECTION, "Loading Plugins")
pm = plugin_manager.PluginManager()
failedPlugins = pm.SetListOfEnvironmentDescriptors(
build_env.plugins)
if failedPlugins:
logging.critical("One or more plugins failed to load. Halting build.")
for a in failedPlugins:
logging.error("Failed Plugin: {0}".format(a["name"]))
raise Exception("One or more plugins failed to load.")
helper = HelperFunctions()
if(helper.LoadFromPluginManager(pm) > 0):
raise Exception("One or more helper plugins failed to load.")
# Make a pathobj so we can normalize and validate the workspace
# and packages path. The Settings Manager can return absolute or
# relative paths
pathobj = Edk2Path(self.GetWorkspaceRoot(), self.GetPackagesPath())
#
# Now we can actually kick off a build.
#
logging.log(edk2_logging.SECTION, "Kicking off build")
ret = self.PlatformBuilder.Go(pathobj.WorkspacePath,
os.pathsep.join(pathobj.PackagePathList),
helper, pm)
logging.log(edk2_logging.SECTION, f"Log file is located at: {self.log_filename}")
return ret
def test_get_relative_path_when_packages_path_list_contains_substrings(self):
''' test usage of GetEdk2RelativePathFromAbsolutePath when members of PackagePathList contain
substrings of themselves, for example "MU" and "MU_TIANO"
File layout:
root/ <-- current working directory (self.tmp)
folder_ws/ <-- workspace root
folder_pp/ <-- packages path
PPTestPkg1/ <-- A edk2 package
PPTestPkg1.DEC
module1/
module1.INF
X64/
TestFile1.c
folder_pp_suffix/ <-- packages path
PPTestPkg2/ <-- A edk2 package
PPTestPkg2.DEC
module2/
module2.INF
X64/
TestFile2.c
'''
ws_rel = "folder_ws"
ws_abs = os.path.join(self.tmp, ws_rel)
os.mkdir(ws_abs)
folder_pp1_rel = "folder_pp"
folder_pp1_abs = os.path.join(ws_abs, folder_pp1_rel)
os.mkdir(folder_pp1_abs)
folder_pp2_rel = "folder_pp_suffix"
folder_pp2_abs = os.path.join(ws_abs, folder_pp2_rel)
os.mkdir(folder_pp2_abs)
ws_p_name = "PPTestPkg2"
ws_pkg_abs = self._make_edk2_package_helper(folder_pp2_abs, ws_p_name)
pathobj = Edk2Path(ws_abs, [folder_pp1_abs, folder_pp2_abs])
# file in workspace
p = os.path.join(ws_pkg_abs, "module2", "X64", "TestFile2.c")
self.assertEqual(pathobj.GetEdk2RelativePathFromAbsolutePath(p), f"{ws_p_name}/module2/X64/TestFile2.c")
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
# SET PACKAGE PATH
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# Check Dependencies for Repo
for dependency in self.PlatformSettings.GetDependencies():
pplist.append(dependency["Path"])
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error(
"You need to run stewart_ci_setup to resolve all repos.")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(
f"Package Path: {self.PlatformSettings.GetPackagesPath()}")
# logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version))
# logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
# logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Enviroment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
archSupported = " ".join(self.PlatformSettings.GetArchSupported())
env.SetValue("TARGET_ARCH", archSupported,
"from PlatformSettings.GetArchSupported()")
_targets = " ".join(self.PlatformSettings.GetTargetsSupported())
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
if len(self.packageList) == 0:
self.packageList.extend(self.PlatformSettings.GetPackages())
for pkgToRunOn in self.packageList:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
ts = JunitReport.create_new_testsuite(
pkgToRunOn,
f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}"
)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txthandle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, mdhandle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txthandle, mdhandle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .mu.json
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# check the resulting configuration
config_validator.check_package_confg(pkgToRunOn, pkg_config,
pluginList)
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
for Descriptor in pluginList:
# Get our targets
targets = ["DEBUG"]
if Descriptor.Obj.IsTargetDependent() and _targets:
targets = self.PlatformSettings.GetTargetsSupported()
for target in targets:
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
pkgToRunOn, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# perhaps we should ask the validator to run on the package for this target
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc != 0):
failure_num += 1
if (rc is None):
logging.error(
"--->Test Failed: %s returned NoneType" %
Descriptor.Name)
else:
logging.error(
"--->Test Failed: %s returned %d" %
(Descriptor.Name, rc))
else:
edk2_logging.log_progress(
f"--->Test Success {Descriptor.Name} {target}")
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num
################################################
# This plugin python file is also
# a command line tool
#
################################################
if __name__ == '__main__':
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toollib.uefi.edk2.path_utilities import Edk2Path
# Parse required paths passed from cmd line arguments
Paths = path_parse()
# check if we are asked to update an .inf file "in-place"
if (Paths.RegenPath is not None):
pathtool = Edk2Path(Paths.WorkSpace, Paths.RegenPackagePath)
v1_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?)")
v2_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?)")
with open (Paths.RegenPath) as fd:
RegenInfData = fd.read()
RegenInfOutData = ""
for line in RegenInfData.splitlines (True):
match = v1_regex.match(line)
if match is None:
match = v2_regex.match(line)
if match is not None:
rel_path = match.group(3)
abs_path = pathtool.GetAbsolutePathOnThisSystemFromEdk2RelativePath(rel_path)
def do_pre_build(self, thebuilder):
try:
error_count = 0
'''
# this scans the whole build directory for bmp's
bmp_search_path = os.path.join(thebuilder.ws,"**","*.bmp");
for found_item in glob.iglob(bmp_search_path, recursive=True):
if CheckBmp(found_item):
logging.error("{} failed image check".format(found_item))
error_count += 1
return error_count
'''
fp = FdfParser()
dp = DscParser()
ws = thebuilder.ws
pp = thebuilder.pp.split(";")
edk2 = Edk2Path(ws, pp)
ActiveDsc = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
thebuilder.env.GetValue("ACTIVE_PLATFORM"))
ActiveFdf = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
thebuilder.env.GetValue("FLASH_DEFINITION"))
if ActiveFdf is None:
self.logger.info("No FDF found- BMP check skipped")
return 0
# parse the DSC and the FDF
dp.SetBaseAbsPath(ws).SetPackagePaths(pp)
dp.SetInputVars(thebuilder.env.GetAllBuildKeyValues()).ParseFile(
ActiveDsc) # parse the DSC for build vars
fp.SetBaseAbsPath(ws).SetPackagePaths(pp)
fp.SetInputVars(dp.LocalVars).ParseFile(
ActiveFdf) # give FDF parser the vars from DSC
# for each FV section in the DSC
for FV_name in fp.FVs:
FV_files = fp.FVs[FV_name]["Files"]
# now look for images in each file of this FV
for fv_file_name in FV_files:
fv_file = FV_files[fv_file_name]
if fv_file["type"].upper() != 'FREEFORM':
continue
fv_file_raw = fv_file['RAW']
fv_file_raw_list = []
if isinstance(fv_file_raw, list):
fv_file_raw_list = fv_file_raw
else:
fv_file_raw_list.append(fv_file_raw)
# for each file that is RAW type
for fv_file_raw_item in fv_file_raw_list:
# check if it ends with a bmp
if fv_file_raw_item.lower().endswith(".bmp"):
logging.debug(fv_file_raw_item)
BmpFilePath = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
fv_file_raw_item)
logging.debug(BmpFilePath)
if BmpCheckPlugin.CheckBmp(
BmpFilePath): # do the check
self.logger.error(
"{} failed image check".format(
fv_file_raw_item))
error_count += 1
return error_count
except:
self.logger.warning(
"Unable to read the FDF. Please update your Edk2-Pytools-* Packages"
)
return 0
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
Edk2CiBuild.collect_python_pip_info()
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error("Src Tree is invalid. Did you Setup correctly?")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(f"Package Path: {pplist}")
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Environment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls.
# This would be better in an edk2 plugin so that it could be modified/controlled
# more easily
#
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list),
"from edk2 ci build.py")
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
for pkgToRunOn in self.requested_package_list:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}"
ts = JunitReport.create_new_testsuite(pkgToRunOn,
package_class_name)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txt_handle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, md_handle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txt_handle, md_handle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .ci.yaml
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
# For each plugin
for Descriptor in pluginList:
# For each target
for target in self.requested_target_list:
if (target not in Descriptor.Obj.RunsOnTargetList()):
continue
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
package_class_name, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = self.merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/package path relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc > 0):
failure_num += 1
if (rc is None):
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned NoneType"
)
else:
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned {rc}"
)
elif (rc < 0):
logging.warn(
f"--->Test Skipped: in plugin! {Descriptor.Name} {target}"
)
else:
edk2_logging.log_progress(
f"--->Test Success: {Descriptor.Name} {target}"
)
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num
def do_post_build(self, thebuilder):
starttime = datetime.now()
logging.info(
"---------------------------------------------------------")
logging.info(
"-----------Postbuild Image Validation Starting-----------")
logging.info(
"---------------------------------------------------------")
# Load Configuration Data
config_path = thebuilder.env.GetValue("PE_VALIDATION_PATH", None)
tool_chain_tag = thebuilder.env.GetValue("TOOL_CHAIN_TAG")
if config_path is None:
logging.info(
"PE_VALIDATION_PATH not set, PE Image Validation Skipped")
return 0 # Path not set, Plugin skipped
if not os.path.isfile(config_path):
logging.error("Invalid PE_VALIDATION_PATH. File not Found")
return 1
with open(config_path) as jsonfile:
config_data = json.load(jsonfile)
self.test_manager.config_data = config_data
self.config_data = config_data
self.ignore_list = config_data["IGNORE_LIST"]
self.arch_dict = config_data["TARGET_ARCH"]
count = 0
# Start Pre-Compiled Image Verification
fdf_parser = FdfParser()
dsc_parser = DscParser()
ws = thebuilder.ws
pp = thebuilder.pp.split(os.pathsep)
edk2 = Edk2Path(ws, pp)
ActiveDsc = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
thebuilder.env.GetValue("ACTIVE_PLATFORM"))
ActiveFdf = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
thebuilder.env.GetValue("FLASH_DEFINITION"))
if ActiveFdf is None:
logging.info("No FDF found - PE Image Validation skipped")
return 0
# parse the DSC and the FDF
dsc_parser.SetBaseAbsPath(ws).SetPackagePaths(pp)
dsc_parser.SetInputVars(
thebuilder.env.GetAllBuildKeyValues()).ParseFile(
ActiveDsc) # parse the DSC for build vars
fdf_parser.SetBaseAbsPath(ws).SetPackagePaths(pp)
fdf_parser.SetInputVars(dsc_parser.LocalVars).ParseFile(
ActiveFdf) # give FDF parser the vars from DSC
# Test all pre-compiled efis described in the fdf
result = Result.PASS
for FV_name in fdf_parser.FVs: # Get all Firmware volumes
FV_files = fdf_parser.FVs[FV_name]["Files"]
for fv_file_name in FV_files: # Iterate over each file in the firmware volume
fv_file = FV_files[fv_file_name]
if "PE32" in fv_file: # Any PE32 section in the FV contains a path to the efi
# could have multiple PE32 sections
for efi_path in fv_file["PE32"]:
efi_path = self._resolve_vars(thebuilder, efi_path)
efi_path = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
efi_path)
if efi_path == None:
logging.warn(
"Unable to parse the path to the pre-compiled efi"
)
continue
if os.path.basename(efi_path) in self.ignore_list:
continue
logging.info(
f'Performing Image Verification ... {os.path.basename(efi_path)}'
)
if self._validate_image(
efi_path, fv_file["type"]) == Result.FAIL:
result = Result.FAIL
count += 1
# End Pre-Compiled Image Verification
# Start Build Time Compiled Image Verification
result = Result.PASS
for arch in thebuilder.env.GetValue("TARGET_ARCH").split():
efi_path_list = self._walk_directory_for_extension(
['.efi'],
f'{thebuilder.env.GetValue("BUILD_OUTPUT_BASE")}/{arch}')
for efi_path in efi_path_list:
if os.path.basename(efi_path) in self.ignore_list:
continue
# Perform Image Verification on any output efi's
# Grab profile from makefile
if efi_path.__contains__("OUTPUT"):
try:
if tool_chain_tag.__contains__("VS"):
profile = self._get_profile_from_makefile(
f'{Path(efi_path).parent.parent}/Makefile')
elif tool_chain_tag.__contains__("GCC"):
profile = self._get_profile_from_makefile(
f'{Path(efi_path).parent.parent}/GNUmakefile')
elif tool_chain_tag.__contains__("CLANG"):
profile = self._get_profile_from_makefile(
f'{Path(efi_path).parent.parent}/GNUmakefile')
else:
logging.warn(
"Unexpected TOOL_CHAIN_TAG... Cannot parse makefile. Using DEFAULT profile."
)
profile = "DEFAULT"
except:
logging.warn(
f'Failed to parse makefile at [{Path(efi_path).parent.parent}/GNUmakefile]'
)
logging.warn(f'Using DEFAULT profile')
profile = "DEFAULT"
logging.info(
f'Performing Image Verification ... {os.path.basename(efi_path)}'
)
if self._validate_image(efi_path, profile) == Result.FAIL:
result = Result.FAIL
count += 1
# End Built Time Compiled Image Verification
endtime = datetime.now()
delta = endtime - starttime
logging.info(
"---------------------------------------------------------")
logging.info(
"-----------Postbuild Image Validation Finished-----------")
logging.info(
"------------------{:04d} Images Verified-------------------".
format(count))
logging.info(
"-------------- Running Time (mm:ss): {0[0]:02}:{0[1]:02} --------------"
.format(divmod(delta.seconds, 60)))
logging.info(
"---------------------------------------------------------")
if result == Result.FAIL:
return 1
else:
return 0
################################################
# This plugin python file is also
# a command line tool
#
################################################
if __name__ == '__main__':
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toollib.uefi.edk2.path_utilities import Edk2Path
# Parse required paths passed from cmd line arguments
Paths = path_parse()
dummy_list = []
pathtool = Edk2Path(Paths.WorkSpace, dummy_list)
# Use absolute module path to find package path
pkg_path = pathtool.GetContainingPackage(Paths.ModulePath)
rel_path = Paths.ModulePath[Paths.ModulePath.find(pkg_path):]
rel_path = rel_path.replace('\\', '/')
mod_hash = ModuleHashCal(Paths.ModulePath)
VERSION_INDEX = Paths.Version - 1
if VERSION_INDEX == 0:
print(
"Copy and paste the following line(s) to your overrider inf file(s):\n"
)
print('#Override : %08d | %s | %s | %s' %
def test_nonexistant_ws(self):
''' test edk2path with invalid workspace'''
invalid_ws = os.path.join(self.tmp, "invalidpath")
with self.assertRaises(Exception):
Edk2Path(invalid_ws, [])
def test_get_containing_package_outside_workspace(self):
''' test basic usage of GetContainingPackage with packages path
outside the workspace
File layout:
root/ <-- current working directory (self.tmp)
folder_ws/ <-- workspace root
WSTestPkg/ <-- A edk2 package
WSTestPkg.dec
module1/
module1.inf
module2/
module2.inf
X64/
TestFile.c
folder_pp/ <-- packages path
PPTestPkg/ <-- A edk2 package
PPTestPkg.DEC
module1/
module1.INF
module2/
module2.INF
X64/
TestFile.c
'''
ws_rel = "folder_ws"
ws_abs = os.path.join(self.tmp, ws_rel)
os.mkdir(ws_abs)
folder_pp_rel = "pp1"
folder_pp1_abs = os.path.join(self.tmp, folder_pp_rel)
os.mkdir(folder_pp1_abs)
ws_p_name = "WSTestPkg"
ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name)
pp_p_name = "PPTestPkg"
pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False)
pathobj = Edk2Path(ws_abs, [folder_pp1_abs])
# file in WSTestPkg root
p = os.path.join(ws_pkg_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name)
# file in module in WSTestPkg
p = os.path.join(ws_pkg_abs, "module1", "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name)
# file in workspace root - no package- should return ws root
p = os.path.join(ws_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), ws_rel)
# file outside of the workspace - invalid and should return None
p = os.path.join(os.path.dirname(ws_abs), "testfile.c")
self.assertIsNone(pathobj.GetContainingPackage(p))
# file in PPTestPkg root
p = os.path.join(pp_pkg_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name)
# file in module in WSTestPkg
p = os.path.join(pp_pkg_abs, "module1", "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name)
# file in packages path root - no package- should return packages path dir
p = os.path.join(folder_pp1_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), folder_pp_rel)
def test_basic_init_ws_abs(self):
''' test edk2path with valid absolute path to workspace'''
pathobj = Edk2Path(self.tmp, [])
self.assertEqual(pathobj.WorkspacePath, self.tmp)
def test_get_containing_package_ws_abs_different_case(self):
''' test basic usage of GetContainingPackage when the workspace path has different case for
the drive letter then the incoming paths. This can happen on Windows
if os.path.realpath is used.
File layout:
root/ <-- current working directory (self.tmp)
folder_ws/ <-- workspace root
folder_pp/ <-- packages path
PPTestPkg/ <-- A edk2 package
PPTestPkg.DEC
module1/
module1.INF
module2/
module2.INF
X64/
TestFile.c
WSTestPkg/ <-- A edk2 package
WSTestPkg.dec
module1/
module1.inf
module2/
module2.inf
X64/
TestFile.c
'''
ws_rel = "folder_ws"
ws_abs = os.path.join(self.tmp, ws_rel)
wsi_abs = os.path.join(self.tmp, ws_rel.capitalize()) # invert the case of the first char of the ws folder name
os.mkdir(ws_abs)
folder_pp_rel = "pp1"
folder_pp1_abs = os.path.join(ws_abs, folder_pp_rel)
os.mkdir(folder_pp1_abs)
ws_p_name = "WSTestPkg"
ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name)
pp_p_name = "PPTestPkg"
pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False)
pathobj = Edk2Path(wsi_abs, [folder_pp1_abs])
# confirm inverted
self.assertNotEqual(pathobj.WorkspacePath, ws_abs)
self.assertEqual(os.path.normcase(pathobj.WorkspacePath), os.path.normcase(ws_abs))
# file in WSTestPkg root
p = os.path.join(ws_pkg_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name)
# file in module in WSTestPkg
p = os.path.join(ws_pkg_abs, "module1", "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name)
# file in workspace root - no package- should return ws root
p = os.path.join(ws_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), ws_rel)
# file outside of the workspace - invalid and should return None
p = os.path.join(os.path.dirname(ws_abs), "testfile.c")
self.assertIsNone(pathobj.GetContainingPackage(p))
# file in PPTestPkg root
p = os.path.join(pp_pkg_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name)
# file in module in WSTestPkg
p = os.path.join(pp_pkg_abs, "module1", "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name)
# file in packages path root - no package- should return packages path dir
p = os.path.join(folder_pp1_abs, "testfile.c")
self.assertEqual(pathobj.GetContainingPackage(p), folder_pp_rel)
def test_get_containing_module(self):
''' test basic usage of GetContainingModule with packages path nested
inside the workspace
File layout:
root/ <-- current working directory (self.tmp)
folder_ws/ <-- workspace root
folder_pp/ <-- packages path
PPTestPkg/ <-- A edk2 package
PPTestPkg.DEC
module1/
module1.INF
module2/
module2.INF
X64/
TestFile.c
WSTestPkg/ <-- A edk2 package
WSTestPkg.dec
module1/
module1.inf
module2/
module2.inf
X64/
TestFile.c
'''
ws_rel = "folder_ws"
ws_abs = os.path.join(self.tmp, ws_rel)
os.mkdir(ws_abs)
folder_pp_rel = "pp1"
folder_pp1_abs = os.path.join(ws_abs, folder_pp_rel)
os.mkdir(folder_pp1_abs)
ws_p_name = "WSTestPkg"
ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name)
pp_p_name = "PPTestPkg"
pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False)
pathobj = Edk2Path(ws_abs, [folder_pp1_abs])
# file in WSTestPkg root. Module list is empty
p = os.path.join(ws_pkg_abs, "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file in module in WSTestPkg/module1/module1.inf
p = os.path.join(ws_pkg_abs, "module1", "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 1)
self.assertIn(os.path.join(ws_pkg_abs, "module1", "module1.inf"), relist)
# file in workspace root - no package- should return ws root
p = os.path.join(ws_abs, "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file outside of the workspace - invalid and should return None
p = os.path.join(os.path.dirname(ws_abs), "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file in module2 x64
p = os.path.join(ws_pkg_abs, "module2", "X64", "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 1)
self.assertIn(os.path.join(ws_pkg_abs, "module2", "module2.inf"), relist)
# inf file in module2 x64
p = os.path.join(ws_pkg_abs, "module2", "module2.inf")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 1)
self.assertIn(os.path.join(ws_pkg_abs, "module2", "module2.inf"), relist)
# file in PPTestPkg root
p = os.path.join(pp_pkg_abs, "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file in module in PPTestPkg
p = os.path.join(pp_pkg_abs, "module1", "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 1)
self.assertIn(os.path.join(pp_pkg_abs, "module1", "module1.INF"), relist)
# inf file in module in PPTestPkg
p = os.path.join(pp_pkg_abs, "module1", "module1.INF")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 1)
self.assertIn(os.path.join(pp_pkg_abs, "module1", "module1.INF"), relist)
# file in packages path root - no module
p = os.path.join(folder_pp1_abs, "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file doesn't exist and parent folder doesn't exist
p = os.path.join(ws_pkg_abs, "ThisParentDirDoesNotExist", "ThisFileDoesNotExist.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file doesn't exist and parent folder doesn't exist and parent parent folder doesn't exist
p = os.path.join(ws_pkg_abs, "DirDirDoesNotExist", "DirDoesNotExist", "FileDoesNotExist.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 0)
# file doesn't exist and parent folder doesn't exist but parent parent is valid module
# file in module in WSTestPkg/module1/module1.inf
p = os.path.join(ws_pkg_abs, "module1", "ThisParentDirDoesNotExist", "testfile.c")
relist = pathobj.GetContainingModules(p)
self.assertEqual(len(relist), 1)
self.assertIn(os.path.join(ws_pkg_abs, "module1", "module1.inf"), relist)
