def test_dsc_parse_file_on_package_path(self):
''' This tests whether includes work properly if no fail mode is on'''
workspace = tempfile.mkdtemp()
working_dir_name = "working"
working2_dir_name = "working2"
working_folder = os.path.join(workspace, working_dir_name)
working2_folder = os.path.join(working_folder, working2_dir_name)
os.makedirs(working_folder, exist_ok=True)
os.makedirs(working2_folder, exist_ok=True)
file1_name = "file1.dsc"
file1_path = os.path.join(working2_folder, file1_name)
file1_short_path = os.path.join(working2_dir_name, file1_name)
file1_data = "[Defines]\n INCLUDED=TRUE"
TestDscParserIncludes.write_to_file(file1_path, file1_data)
with self.assertRaises(FileNotFoundError):
parser = DscParser()
parser.SetBaseAbsPath(workspace)
parser.ParseFile(file1_short_path)
parser = DscParser()
parser.SetBaseAbsPath(workspace)
parser.SetPackagePaths([
working_folder,
])
parser.ParseFile(file1_short_path)
self.assertEqual(parser.LocalVars["INCLUDED"],
"TRUE") # make sure we got the defines
def test_dsc_include_single_file(self):
''' This tests whether includes work properly '''
workspace = tempfile.mkdtemp()
file1_name = "file1.dsc"
file2_name = "file2.dsc"
file1_path = os.path.join(workspace, file1_name)
file2_path = os.path.join(workspace, file2_name)
file1_data = f"!include {file2_name}"
file2_data = "[Defines]\nINCLUDED = TRUE"
TestDscParserIncludes.write_to_file(file1_path, file1_data)
TestDscParserIncludes.write_to_file(file2_path, file2_data)
parser = DscParser()
parser.SetBaseAbsPath(workspace)
parser.ParseFile(file1_path)
# test to make sure we did it right
self.assertEqual(len(parser.LocalVars),
1) # make sure we only have one define
self.assertEqual(parser.LocalVars["INCLUDED"],
"TRUE") # make sure we got the defines
self.assertEqual(len(parser.GetAllDscPaths()),
2) # make sure we have two dsc paths
def ParseDscFile(self):
if self.env.GetValue("ACTIVE_PLATFORM") is None:
logging.error(
"The DSC file was not set. Please set ACTIVE_PLATFORM")
return -1
dsc_file_path = self.mws.join(self.ws,
self.env.GetValue("ACTIVE_PLATFORM"))
if (os.path.isfile(dsc_file_path)):
# parse DSC File
logging.debug(
"Parse Active Platform DSC file: {0}".format(dsc_file_path))
# Get the vars from the environment that are not build keys
input_vars = self.env.GetAllNonBuildKeyValues()
# Update with special environment set build keys
input_vars.update(self.env.GetAllBuildKeyValues())
dscp = DscParser().SetBaseAbsPath(self.ws).SetPackagePaths(
self.pp.split(os.pathsep)).SetInputVars(input_vars)
dscp.ParseFile(dsc_file_path)
for key, value in dscp.LocalVars.items():
# set env as overrideable
self.env.SetValue(key, value, "From Platform DSC File", True)
else:
logging.error("Failed to find DSC file")
return -1
return 0
def test_dsc_include_missing_file(self):
''' This tests whether includes work properly '''
workspace = tempfile.gettempdir()
file1_name = "file1.dsc"
file1_path = os.path.join(workspace, file1_name)
file1_data = f"!include BAD_FILE.dsc"
TestDscParserIncludes.write_to_file(file1_path, file1_data)
parser = DscParser()
parser.SetBaseAbsPath(workspace)
with self.assertRaises(FileNotFoundError):
parser.ParseFile(file1_path)
def test_dsc_include_missing_file_no_fail_mode(self):
''' This tests whether includes work properly if no fail mode is on'''
workspace = tempfile.mkdtemp()
file1_name = "file1.dsc"
file1_path = os.path.join(workspace, file1_name)
file1_data = "!include BAD_FILE.dsc"
TestDscParserIncludes.write_to_file(file1_path, file1_data)
parser = DscParser()
parser.SetNoFailMode()
parser.SetBaseAbsPath(workspace)
parser.ParseFile(file1_path)
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
self._env = environment
# Parse the config for required DscPath element
if "DscPath" not in pkgconfig:
tc.SetSkipped()
tc.LogStdError("DscPath not found in config file. Nothing to compile.")
return -1
AP = Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(packagename)
APDSC = os.path.join(AP, pkgconfig["DscPath"].strip())
AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC)
if AP is None or AP_Path is None or not os.path.isfile(APDSC):
tc.SetSkipped()
tc.LogStdError("Package Dsc not found.")
return -1
logging.info("Building {0}".format(AP_Path))
self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin")
# Parse DSC to check for SUPPORTED_ARCHITECTURES
dp = DscParser()
dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
dp.SetPackagePaths(Edk2pathObj.PackagePathList)
dp.ParseFile(AP_Path)
if "SUPPORTED_ARCHITECTURES" in dp.LocalVars:
SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|')
TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(' ')
# Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES
if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0:
tc.SetSkipped()
tc.LogStdError("No supported architecutres to build")
return -1
uefiBuilder = UefiBuilder()
# do all the steps
# WorkSpace, PackagesPath, PInHelper, PInManager
ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM)
if ret != 0: # failure:
tc.SetFailed("Compile failed for {0}".format(packagename), "Compile_FAILED")
tc.LogStdError("{0} Compile failed with error code {1} ".format(AP_Path, ret))
return 1
else:
tc.SetSuccess()
return 0
def test_dsc_include_relative_path(self):
''' This tests whether includes work properly with a relative path'''
workspace = tempfile.mkdtemp()
outside_folder = os.path.join(workspace, "outside")
inside_folder = os.path.join(outside_folder, "inside")
inside2_folder = os.path.join(outside_folder, "inside2")
random_folder = os.path.join(outside_folder, "random")
os.makedirs(inside_folder, exist_ok=True)
os.makedirs(inside2_folder, exist_ok=True)
os.makedirs(random_folder, exist_ok=True)
cwd = os.getcwd()
os.chdir(random_folder)
try:
file1_name = "file1.dsc"
file1_path = os.path.join(outside_folder, file1_name)
file2_name = "file2.dsc"
file2_path = os.path.join(inside_folder, file2_name)
file3_name = "file3.dsc"
file3_path = os.path.join(inside2_folder, file3_name)
file1_data = "!include " + os.path.relpath(
file2_path, os.path.dirname(file1_path)).replace("\\", "/")
file2_data = "!include " + os.path.relpath(
file3_path, os.path.dirname(file2_path)).replace("\\", "/")
file3_data = "[Defines]\n INCLUDED=TRUE"
print(f"{file1_path}: {file1_data}")
print(f"{file2_path}: {file2_data}")
print(f"{file3_path}: {file3_data}")
TestDscParserIncludes.write_to_file(file1_path, file1_data)
TestDscParserIncludes.write_to_file(file2_path, file2_data)
TestDscParserIncludes.write_to_file(file3_path, file3_data)
parser = DscParser()
parser.SetBaseAbsPath(workspace)
parser.ParseFile(file1_path)
self.assertEqual(parser.LocalVars["INCLUDED"],
"TRUE") # make sure we got the defines
finally:
os.chdir(cwd)
def ParseDscFile(self):
dsc_file_path = self.mws.join(self.ws,
self.env.GetValue("ACTIVE_PLATFORM"))
if (os.path.isfile(dsc_file_path)):
# parse DSC File
logging.debug(
"Parse Active Platform DSC file: {0}".format(dsc_file_path))
dscp = DscParser().SetBaseAbsPath(self.ws).SetPackagePaths(
self.pp.split(os.pathsep)).SetInputVars(
self.env.GetAllBuildKeyValues())
dscp.ParseFile(dsc_file_path)
for key, value in dscp.LocalVars.items():
# set env as overrideable
self.env.SetValue(key, value, "From Platform DSC File", True)
else:
logging.error("Failed to find DSC file")
return -1
return 0
def ParseFdfFile(self):
if (self.env.GetValue("FLASH_DEFINITION") is None):
logging.debug("No flash definition set")
return 0
if (os.path.isfile(
self.mws.join(self.ws,
self.env.GetValue("FLASH_DEFINITION")))):
# parse the FDF file- fdf files have similar syntax to DSC and therefore parser works for both.
logging.debug("Parse Active Flash Definition (FDF) file")
fdf_parser = DscParser().SetBaseAbsPath(self.ws).SetPackagePaths(
self.pp.split(os.pathsep)).SetInputVars(
self.env.GetAllBuildKeyValues())
pa = self.mws.join(self.ws, self.env.GetValue("FLASH_DEFINITION"))
fdf_parser.ParseFile(pa)
for key, value in fdf_parser.LocalVars.items():
self.env.SetValue(key, value, "From Platform FDF File", True)
else:
logging.error("Failed to find FDF file")
return -2
return 0
def RunBuildPlugin(self,
packagename,
Edk2pathObj,
pkgconfig,
environment,
PLM,
PLMHelper,
tc,
output_stream=None):
overall_status = 0
# Parse the config for required DscPath element
if "DscPath" not in pkgconfig:
tc.SetSkipped()
tc.LogStdError(
"DscPath not found in config file. Nothing to check.")
return -1
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
packagename)
abs_dsc_path = os.path.join(abs_pkg_path, pkgconfig["DscPath"].strip())
wsr_dsc_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(
abs_dsc_path)
if abs_dsc_path is None or wsr_dsc_path == "" or not os.path.isfile(
abs_dsc_path):
tc.SetSkipped()
tc.LogStdError("Package Host Unit Test Dsc not found")
return 0
# Get INF Files
INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path)
INFFiles = [
Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x)
for x in INFFiles
] # make edk2relative path so can compare with DSC
# remove ignores
if "IgnoreInf" in pkgconfig:
for a in pkgconfig["IgnoreInf"]:
a = a.replace(os.sep, "/")
try:
tc.LogStdOut("Ignoring INF {0}".format(a))
INFFiles.remove(a)
except:
tc.LogStdError(
"HostUnitTestDscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file"
.format(a))
logging.info(
"HostUnitTestDscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file"
.format(a))
# DSC Parser
dp = DscParser()
dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
dp.SetPackagePaths(Edk2pathObj.PackagePathList)
dp.SetInputVars(environment.GetAllBuildKeyValues())
dp.ParseFile(wsr_dsc_path)
# Check if INF in component section
for INF in INFFiles:
if not any(INF.strip() in x for x in dp.ThreeMods) and \
not any(INF.strip() in x for x in dp.SixMods) and \
not any(INF.strip() in x for x in dp.OtherMods):
infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath)
infp.SetPackagePaths(Edk2pathObj.PackagePathList)
infp.ParseFile(INF)
if ("MODULE_TYPE" not in infp.Dict):
tc.LogStdOut(
"Ignoring INF. Missing key for MODULE_TYPE {0}".format(
INF))
continue
if (infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"):
# should compile test a library that is declared type HOST_APPLICATION
pass
elif len(infp.SupportedPhases) > 0 and \
"HOST_APPLICATION" in infp.SupportedPhases:
# should compile test a library that supports HOST_APPLICATION but
# require it to be an explicit opt-in
pass
else:
tc.LogStdOut(
"Ignoring INF. MODULE_TYPE or suppored phases not HOST_APPLICATION {0}"
.format(INF))
continue
logging.critical(INF + " not in " + wsr_dsc_path)
tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path))
overall_status = overall_status + 1
# If XML object exists, add result
if overall_status != 0:
tc.SetFailed(
"HostUnitTestDscCompleteCheck {0} Failed. Errors {1}".format(
wsr_dsc_path, overall_status), "CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status
def RunBuildPlugin(self,
packagename,
Edk2pathObj,
pkgconfig,
environment,
PLM,
PLMHelper,
tc,
output_stream=None):
self._env = environment
environment.SetValue("CI_BUILD_TYPE", "host_unit_test",
"Set in HostUnitTestCompilerPlugin")
# Parse the config for required DscPath element
if "DscPath" not in pkgconfig:
tc.SetSkipped()
tc.LogStdError(
"DscPath not found in config file. Nothing to compile for HostBasedUnitTests."
)
return -1
AP = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
packagename)
APDSC = os.path.join(AP, pkgconfig["DscPath"].strip())
AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC)
if AP is None or AP_Path is None or not os.path.isfile(APDSC):
tc.SetSkipped()
tc.LogStdError("Package HostBasedUnitTest Dsc not found.")
return -1
logging.info("Building {0}".format(AP_Path))
self._env.SetValue("ACTIVE_PLATFORM", AP_Path,
"Set in Compiler Plugin")
num, RUNNABLE_ARCHITECTURES = self.__GetHostUnitTestArch(environment)
if (num == 0):
tc.SetSkipped()
tc.LogStdError("No host architecture compatibility")
return -1
if not environment.SetValue(
"TARGET_ARCH", RUNNABLE_ARCHITECTURES,
"Update Target Arch based on Host Support"):
#use AllowOverride function since this is a controlled attempt to change
environment.AllowOverride("TARGET_ARCH")
if not environment.SetValue(
"TARGET_ARCH", RUNNABLE_ARCHITECTURES,
"Update Target Arch based on Host Support"):
raise RuntimeError("Can't Change TARGET_ARCH as required")
# Parse DSC to check for SUPPORTED_ARCHITECTURES
dp = DscParser()
dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
dp.SetPackagePaths(Edk2pathObj.PackagePathList)
dp.ParseFile(AP_Path)
if "SUPPORTED_ARCHITECTURES" in dp.LocalVars:
SUPPORTED_ARCHITECTURES = dp.LocalVars[
"SUPPORTED_ARCHITECTURES"].split('|')
TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(
' ')
# Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES
if len(set(SUPPORTED_ARCHITECTURES)
& set(TARGET_ARCHITECTURES)) == 0:
tc.SetSkipped()
tc.LogStdError(
"No supported architecutres to build for host unit tests")
return -1
uefiBuilder = UefiBuilder()
# do all the steps
# WorkSpace, PackagesPath, PInHelper, PInManager
ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath,
os.pathsep.join(Edk2pathObj.PackagePathList),
PLMHelper, PLM)
if ret != 0: # failure:
tc.SetFailed("Compile failed for {0}".format(packagename),
"Compile_FAILED")
tc.LogStdError("{0} Compile failed with error code {1} ".format(
AP_Path, ret))
return 1
else:
tc.SetSuccess()
return 0
def get_packages_to_build(self, possible_packages: list) -> dict:
self.parsed_dec_cache = {}
(rc, files) = self._get_files_that_changed_in_this_pr(self.pr_target)
if rc != 0:
return {}
remaining_packages = possible_packages.copy(
) # start with all possible packages and remove each
# package once it is determined to be build. This optimization
# avoids checking packages that already will be built.
packages_to_build = {
} # packages to build. Key = pkg name, Value = 1st reason for build
#
# Policy 1 - CI Settings file defined
#
after = self.PlatformSettings.FilterPackagesToTest(
files, remaining_packages)
for a in after:
if a not in remaining_packages:
raise ValueError(
f"PlatformSettings.FilterPackagesToTest returned package not allowed {a}"
)
packages_to_build[
a] = "Policy 1 - PlatformSettings - Filter Packages"
remaining_packages.remove(a)
#
# Policy 2: Build any package that has changed
#
for f in files:
try:
pkg = self.edk2_path_obj.GetContainingPackage(
os.path.abspath(f))
except Exception as e:
self.logger.error(
f"Failed to get package for file {f}. Exception {e}")
# Ignore a file in which we fail to get the package
continue
if (pkg not in packages_to_build.keys()
and pkg in remaining_packages):
packages_to_build[
pkg] = "Policy 2 - Build any package that has changed"
remaining_packages.remove(pkg)
#
# Policy 3: If a file change is a public file then build all packages that
# are dependent on that package.
#
# list of packages with public files that have changed
public_package_changes = []
# Get all public files in packages
for f in files:
try:
pkg = self.edk2_path_obj.GetContainingPackage(
os.path.abspath(f))
except Exception as e:
self.logger.error(
f"Failed to get package for file {f}. Exception {e}")
# Ignore a file in which we fail to get the package
continue
if self._is_public_file(f):
public_package_changes.append(pkg)
# de-duplicate list
public_package_changes = list(set(public_package_changes))
# Now check all remaining packages to see if they depend on the set of packages
# with public file changes.
# NOTE: future enhancement could be to check actual file dependencies
for a in public_package_changes:
for p in remaining_packages[:]: # slice so we can delete as we go
if (self._does_pkg_depend_on_package(p, a)):
packages_to_build[p] = f"Policy 3 - Package depends on {a}"
remaining_packages.remove(
p) # remove from remaining packages
#
# Policy 4: If a file changed in a module and that module is used in the provided dsc file
# then the package of the dSC file must be built
#
PlatformDscInfo = self.PlatformSettings.GetPlatformDscAndConfig()
if PlatformDscInfo is not None and len(remaining_packages) > 0:
if len(remaining_packages) != 1:
raise Exception(
"Policy 4 can only be used by builds for a single package")
# files are all the files changed edk2 workspace root relative path
changed_modules = self._get_unique_module_infs_changed(files)
# now check DSC
dsc = DscParser()
dsc.SetBaseAbsPath(self.edk2_path_obj.WorkspacePath)
dsc.SetPackagePaths(self.edk2_path_obj.PackagePathList)
dsc.SetInputVars(PlatformDscInfo[1])
dsc.ParseFile(PlatformDscInfo[0])
allinfs = dsc.OtherMods + dsc.ThreeMods + dsc.SixMods + dsc.Libs # get list of all INF files
#
# Note: for now we assume that remaining_packages has only 1 package and that it corresponds
# to the DSC file provided.
#
for p in remaining_packages[:]: # slice so we can delete as we go
for cm in changed_modules:
if cm in allinfs: # is the changed module listed in the DSC file?
packages_to_build[
p] = f"Policy 4 - Package Dsc depends on {cm}"
remaining_packages.remove(
p) # remove from remaining packages
break
return packages_to_build