def test_pp_outside_workspace(self): ''' test with packagespath pointing to folder outside of workspace root/ <-- current working directory folder_ws/ <-- workspace root ws packages here folder_pp/ <-- packages path pp packages here ''' ws_rel = "folder_ws" ws_abs = os.path.join(self.tmp, ws_rel) os.mkdir(ws_abs) folder_pp_rel = "pp1" folder_pp1_abs = os.path.join(self.tmp, folder_pp_rel) os.mkdir(folder_pp1_abs) # pp absolute pathobj = Edk2Path(ws_abs, [folder_pp1_abs]) self.assertEqual(pathobj.WorkspacePath, ws_abs) self.assertEqual(pathobj.PackagePathList[0], folder_pp1_abs) # pp relative to cwd pathobj = Edk2Path(ws_abs, [folder_pp_rel]) self.assertEqual(pathobj.WorkspacePath, ws_abs) self.assertEqual(pathobj.PackagePathList[0], folder_pp1_abs)
def test_nonexistant_abs(self): ''' test edk2path with valid ws but invalid pp''' pp = "doesnot_exist" pp_full_1 = os.path.join(self.tmp, pp) # absolute path with self.assertRaises(Exception): Edk2Path(self.tmp, [pp_full_1]) # relative path with self.assertRaises(Exception): Edk2Path(self.tmp, [pp]) # confirm optional parameter to remove invalid pp values pathobj = Edk2Path(self.tmp, [pp], error_on_invalid_pp=False) self.assertEqual(len(pathobj.PackagePathList), 0)
def override_plat_validate(self, thebuilder): result = self.OverrideResult.OR_ALL_GOOD InfFileList = self.get_dsc_inf_list(thebuilder) ws = thebuilder.ws pp = thebuilder.pp.split(os.pathsep) self.PathTool = Edk2Path(ws, pp) if (InfFileList == []): return result modulelist = [] status = [0, 0] # Search through the workspace and package paths for file in InfFileList: temp_list = [] modulenode = self.ModuleNode(file, self.OverrideResult.OR_ALL_GOOD, 0) fullpath = thebuilder.mws.join(thebuilder.ws, file) m_result = self.override_detect_process(thebuilder, fullpath, temp_list, modulenode, status) # Do not log the module that does not have any override records if (len(modulenode.reflist) > 0): modulelist.append(modulenode) if m_result != self.OverrideResult.OR_ALL_GOOD: if m_result != self.OverrideResult.OR_DSC_INF_NOT_FOUND: result = m_result logging.error("Override processing error %s in file %s" % (self.OverrideResult.GetErrStr(m_result), file)) self.override_log_print(thebuilder, modulelist, status) return result
def test_basic_init_ws_cwd(self): ''' test edk2path with a relative path to workspace''' relpath = "testrootfolder" fullpath = os.path.join(self.tmp, relpath) os.mkdir(fullpath) pathobj = Edk2Path(relpath, []) self.assertEqual(pathobj.WorkspacePath, fullpath)
def test_basic_init_ws_abs_different_case(self): inputPath = self.tmp.capitalize() if self.tmp[0].isupper(): inputPath = self.tmp[0].lower() + self.tmp[1:] pathobj = Edk2Path(inputPath, []) self.assertNotEqual(pathobj.WorkspacePath, self.tmp)
def test_get_edk2_relative_path_from_absolute_path(self): ''' test basic usage of GetEdk2RelativePathFromAbsolutePath with packages path nested inside the workspace File layout: root/ <-- current working directory (self.tmp) folder_ws/ <-- workspace root folder_pp/ <-- packages path PPTestPkg/ <-- A edk2 package PPTestPkg.DEC module1/ module1.INF module2/ module2.INF X64/ TestFile.c WSTestPkg/ <-- A edk2 package WSTestPkg.dec module1/ module1.inf module2/ module2.inf X64/ TestFile.c ''' ws_rel = "folder_ws" ws_abs = os.path.join(self.tmp, ws_rel) os.mkdir(ws_abs) folder_pp_rel = "pp1" folder_pp1_abs = os.path.join(ws_abs, folder_pp_rel) os.mkdir(folder_pp1_abs) ws_p_name = "WSTestPkg" ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name) pp_p_name = "PPTestPkg" pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False) pathobj = Edk2Path(ws_abs, [folder_pp1_abs]) # file in packages path p = os.path.join(pp_pkg_abs, "module1", "module1.INF") self.assertEqual(pathobj.GetEdk2RelativePathFromAbsolutePath(p), f"{pp_p_name}/module1/module1.INF") # file in workspace p = os.path.join(ws_pkg_abs, "module2", "X64", "TestFile.c") self.assertEqual(pathobj.GetEdk2RelativePathFromAbsolutePath(p), f"{ws_p_name}/module2/X64/TestFile.c") # file not in workspace p = os.path.join(self.tmp, "module2", "X64", "TestFile.c") self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(p)) # pass in bad parameter self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(None)) # file is outside of code tree and not absolute path p = os.path.join("module2", "X64", "TestFile.c") self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(p)) # file is cwd relative but not absolute path p = os.path.join(ws_rel, ws_p_name, "module2", "X64", "TestFile.c") self.assertIsNone(pathobj.GetEdk2RelativePathFromAbsolutePath(p))
def _initialize_environment_info(self, package_rel_path: str, edk2_path: Edk2Path, package_config: Dict[str, List[str]], tc: JunitReportTestCase) -> None: """ Initializes plugin environment information. """ self._abs_package_path = edk2_path.GetAbsolutePathOnThisSystemFromEdk2RelativePath( package_rel_path) self._abs_workspace_path = edk2_path.WorkspacePath self._package_config = package_config self._package_name = os.path.basename( os.path.normpath(package_rel_path)) self._plugin_name = self.__class__.__name__ self._plugin_path = os.path.dirname(os.path.realpath(__file__)) self._rel_package_path = package_rel_path self._tc = tc
def Go(self): logging.info("Running Python version: " + str(sys.version_info)) Edk2PlatformBuild.collect_python_pip_info() (build_env, shell_env) = self_describing_environment.BootstrapEnvironment( self.GetWorkspaceRoot(), self.GetActiveScopes()) # Bind our current execution environment into the shell vars. ph = os.path.dirname(sys.executable) if " " in ph: ph = '"' + ph + '"' shell_env.set_shell_var("PYTHON_HOME", ph) # PYTHON_COMMAND is required to be set for using edk2 python builds. # todo: work with edk2 to remove the bat file and move to native python calls pc = sys.executable if " " in pc: pc = '"' + pc + '"' shell_env.set_shell_var("PYTHON_COMMAND", pc) # Load plugins logging.log(edk2_logging.SECTION, "Loading Plugins") pm = plugin_manager.PluginManager() failedPlugins = pm.SetListOfEnvironmentDescriptors( build_env.plugins) if failedPlugins: logging.critical("One or more plugins failed to load. Halting build.") for a in failedPlugins: logging.error("Failed Plugin: {0}".format(a["name"])) raise Exception("One or more plugins failed to load.") helper = HelperFunctions() if(helper.LoadFromPluginManager(pm) > 0): raise Exception("One or more helper plugins failed to load.") # Make a pathobj so we can normalize and validate the workspace # and packages path. The Settings Manager can return absolute or # relative paths pathobj = Edk2Path(self.GetWorkspaceRoot(), self.GetPackagesPath()) # # Now we can actually kick off a build. # logging.log(edk2_logging.SECTION, "Kicking off build") ret = self.PlatformBuilder.Go(pathobj.WorkspacePath, os.pathsep.join(pathobj.PackagePathList), helper, pm) logging.log(edk2_logging.SECTION, f"Log file is located at: {self.log_filename}") return ret
def test_get_relative_path_when_packages_path_list_contains_substrings(self): ''' test usage of GetEdk2RelativePathFromAbsolutePath when members of PackagePathList contain substrings of themselves, for example "MU" and "MU_TIANO" File layout: root/ <-- current working directory (self.tmp) folder_ws/ <-- workspace root folder_pp/ <-- packages path PPTestPkg1/ <-- A edk2 package PPTestPkg1.DEC module1/ module1.INF X64/ TestFile1.c folder_pp_suffix/ <-- packages path PPTestPkg2/ <-- A edk2 package PPTestPkg2.DEC module2/ module2.INF X64/ TestFile2.c ''' ws_rel = "folder_ws" ws_abs = os.path.join(self.tmp, ws_rel) os.mkdir(ws_abs) folder_pp1_rel = "folder_pp" folder_pp1_abs = os.path.join(ws_abs, folder_pp1_rel) os.mkdir(folder_pp1_abs) folder_pp2_rel = "folder_pp_suffix" folder_pp2_abs = os.path.join(ws_abs, folder_pp2_rel) os.mkdir(folder_pp2_abs) ws_p_name = "PPTestPkg2" ws_pkg_abs = self._make_edk2_package_helper(folder_pp2_abs, ws_p_name) pathobj = Edk2Path(ws_abs, [folder_pp1_abs, folder_pp2_abs]) # file in workspace p = os.path.join(ws_pkg_abs, "module2", "X64", "TestFile2.c") self.assertEqual(pathobj.GetEdk2RelativePathFromAbsolutePath(p), f"{ws_p_name}/module2/X64/TestFile2.c")
def Go(self): log_directory = os.path.join(self.GetWorkspaceRoot(), self.GetLoggingFolderRelativeToRoot()) # SET PACKAGE PATH # # Get Package Path from config file pplist = self.PlatformSettings.GetPackagesPath( ) if self.PlatformSettings.GetPackagesPath() else [] # Check Dependencies for Repo for dependency in self.PlatformSettings.GetDependencies(): pplist.append(dependency["Path"]) # make Edk2Path object to handle all path operations try: edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist) except Exception as e: logging.error( "You need to run stewart_ci_setup to resolve all repos.") raise e logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}") logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}") logging.info( f"Package Path: {self.PlatformSettings.GetPackagesPath()}") # logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version)) # logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version) # logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version) # Bring up the common minimum environment. logging.log(edk2_logging.SECTION, "Getting Enviroment") (build_env, shell_env) = self_describing_environment.BootstrapEnvironment( self.GetWorkspaceRoot(), self.GetActiveScopes()) env = shell_environment.GetBuildVars() # Bind our current execution environment into the shell vars. ph = os.path.dirname(sys.executable) if " " in ph: ph = '"' + ph + '"' shell_env.set_shell_var("PYTHON_HOME", ph) # PYTHON_COMMAND is required to be set for using edk2 python builds. # todo: work with edk2 to remove the bat file and move to native python calls pc = sys.executable if " " in pc: pc = '"' + pc + '"' shell_env.set_shell_var("PYTHON_COMMAND", pc) archSupported = " ".join(self.PlatformSettings.GetArchSupported()) env.SetValue("TARGET_ARCH", archSupported, "from PlatformSettings.GetArchSupported()") _targets = " ".join(self.PlatformSettings.GetTargetsSupported()) # Generate consumable XML object- junit format JunitReport = JunitTestReport() # Keep track of failures failure_num = 0 total_num = 0 # Load plugins logging.log(edk2_logging.SECTION, "Loading plugins") pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin) if len(self.packageList) == 0: self.packageList.extend(self.PlatformSettings.GetPackages()) for pkgToRunOn in self.packageList: # # run all loaded Edk2CiBuild Plugins/Tests # logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package") logging.info(f"Running on Package: {pkgToRunOn}") ts = JunitReport.create_new_testsuite( pkgToRunOn, f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}" ) packagebuildlog_path = os.path.join(log_directory, pkgToRunOn) _, txthandle = edk2_logging.setup_txt_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) _, mdhandle = edk2_logging.setup_markdown_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) loghandle = [txthandle, mdhandle] shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() # load the package level .mu.json pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath( os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml")) if (pkg_config_file): with open(pkg_config_file, 'r') as f: pkg_config = yaml.safe_load(f) else: logging.info(f"No Pkg Config file for {pkgToRunOn}") pkg_config = dict() # check the resulting configuration config_validator.check_package_confg(pkgToRunOn, pkg_config, pluginList) # get all the defines from the package configuration if "Defines" in pkg_config: for definition_key in pkg_config["Defines"]: definition = pkg_config["Defines"][definition_key] env.SetValue(definition_key, definition, "Edk2CiBuild.py from PkgConfig yaml", False) for Descriptor in pluginList: # Get our targets targets = ["DEBUG"] if Descriptor.Obj.IsTargetDependent() and _targets: targets = self.PlatformSettings.GetTargetsSupported() for target in targets: edk2_logging.log_progress( f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --" ) total_num += 1 shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() env.SetValue("TARGET", target, "Edk2CiBuild.py before RunBuildPlugin") (testcasename, testclassname) = Descriptor.Obj.GetTestName( pkgToRunOn, env) tc = ts.create_new_testcase(testcasename, testclassname) # create the stream for the build log plugin_output_stream = edk2_logging.create_output_stream() # merge the repo level and package level for this specific plugin pkg_plugin_configuration = merge_config( self.PlatformSettings.GetPluginSettings(), pkg_config, Descriptor.descriptor) # perhaps we should ask the validator to run on the package for this target # Still need to see if the package decided this should be skipped if pkg_plugin_configuration is None or\ "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]: tc.SetSkipped() edk2_logging.log_progress( "--->Test Skipped by package! %s" % Descriptor.Name) else: try: # - package is the edk2 path to package. This means workspace/packagepath relative. # - edk2path object configured with workspace and packages path # - any additional command line args # - RepoConfig Object (dict) for the build # - PkgConfig Object (dict) # - EnvConfig Object # - Plugin Manager Instance # - Plugin Helper Obj Instance # - testcase Object used for outputing junit results # - output_stream the StringIO output stream from this plugin rc = Descriptor.Obj.RunBuildPlugin( pkgToRunOn, edk2path, pkg_plugin_configuration, env, self.plugin_manager, self.helper, tc, plugin_output_stream) except Exception as exp: exc_type, exc_value, exc_traceback = sys.exc_info() logging.critical("EXCEPTION: {0}".format(exp)) exceptionPrint = traceback.format_exception( type(exp), exp, exc_traceback) logging.critical(" ".join(exceptionPrint)) tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION") rc = 1 if (rc != 0): failure_num += 1 if (rc is None): logging.error( "--->Test Failed: %s returned NoneType" % Descriptor.Name) else: logging.error( "--->Test Failed: %s returned %d" % (Descriptor.Name, rc)) else: edk2_logging.log_progress( f"--->Test Success {Descriptor.Name} {target}") # revert to the checkpoint we created previously shell_environment.RevertBuildVars() # remove the logger edk2_logging.remove_output_stream(plugin_output_stream) # finished target loop # Finished plugin loop edk2_logging.stop_logging( loghandle) # stop the logging for this particular buildfile shell_environment.RevertBuildVars() # Finished buildable file loop JunitReport.Output( os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml")) # Print Overall Success if (failure_num != 0): logging.error("Overall Build Status: Error") edk2_logging.log_progress( f"There were {failure_num} failures out of {total_num} attempts" ) else: edk2_logging.log_progress("Overall Build Status: Success") return failure_num
################################################ # This plugin python file is also # a command line tool # ################################################ if __name__ == '__main__': from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser from edk2toollib.uefi.edk2.path_utilities import Edk2Path # Parse required paths passed from cmd line arguments Paths = path_parse() # check if we are asked to update an .inf file "in-place" if (Paths.RegenPath is not None): pathtool = Edk2Path(Paths.WorkSpace, Paths.RegenPackagePath) v1_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?)") v2_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?)") with open (Paths.RegenPath) as fd: RegenInfData = fd.read() RegenInfOutData = "" for line in RegenInfData.splitlines (True): match = v1_regex.match(line) if match is None: match = v2_regex.match(line) if match is not None: rel_path = match.group(3) abs_path = pathtool.GetAbsolutePathOnThisSystemFromEdk2RelativePath(rel_path)
def do_pre_build(self, thebuilder): try: error_count = 0 ''' # this scans the whole build directory for bmp's bmp_search_path = os.path.join(thebuilder.ws,"**","*.bmp"); for found_item in glob.iglob(bmp_search_path, recursive=True): if CheckBmp(found_item): logging.error("{} failed image check".format(found_item)) error_count += 1 return error_count ''' fp = FdfParser() dp = DscParser() ws = thebuilder.ws pp = thebuilder.pp.split(";") edk2 = Edk2Path(ws, pp) ActiveDsc = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( thebuilder.env.GetValue("ACTIVE_PLATFORM")) ActiveFdf = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( thebuilder.env.GetValue("FLASH_DEFINITION")) if ActiveFdf is None: self.logger.info("No FDF found- BMP check skipped") return 0 # parse the DSC and the FDF dp.SetBaseAbsPath(ws).SetPackagePaths(pp) dp.SetInputVars(thebuilder.env.GetAllBuildKeyValues()).ParseFile( ActiveDsc) # parse the DSC for build vars fp.SetBaseAbsPath(ws).SetPackagePaths(pp) fp.SetInputVars(dp.LocalVars).ParseFile( ActiveFdf) # give FDF parser the vars from DSC # for each FV section in the DSC for FV_name in fp.FVs: FV_files = fp.FVs[FV_name]["Files"] # now look for images in each file of this FV for fv_file_name in FV_files: fv_file = FV_files[fv_file_name] if fv_file["type"].upper() != 'FREEFORM': continue fv_file_raw = fv_file['RAW'] fv_file_raw_list = [] if isinstance(fv_file_raw, list): fv_file_raw_list = fv_file_raw else: fv_file_raw_list.append(fv_file_raw) # for each file that is RAW type for fv_file_raw_item in fv_file_raw_list: # check if it ends with a bmp if fv_file_raw_item.lower().endswith(".bmp"): logging.debug(fv_file_raw_item) BmpFilePath = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( fv_file_raw_item) logging.debug(BmpFilePath) if BmpCheckPlugin.CheckBmp( BmpFilePath): # do the check self.logger.error( "{} failed image check".format( fv_file_raw_item)) error_count += 1 return error_count except: self.logger.warning( "Unable to read the FDF. Please update your Edk2-Pytools-* Packages" ) return 0
def Go(self): log_directory = os.path.join(self.GetWorkspaceRoot(), self.GetLoggingFolderRelativeToRoot()) Edk2CiBuild.collect_python_pip_info() # # Get Package Path from config file pplist = self.PlatformSettings.GetPackagesPath( ) if self.PlatformSettings.GetPackagesPath() else [] # make Edk2Path object to handle all path operations try: edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist) except Exception as e: logging.error("Src Tree is invalid. Did you Setup correctly?") raise e logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}") logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}") logging.info(f"Package Path: {pplist}") # Bring up the common minimum environment. logging.log(edk2_logging.SECTION, "Getting Environment") (build_env, shell_env) = self_describing_environment.BootstrapEnvironment( self.GetWorkspaceRoot(), self.GetActiveScopes()) env = shell_environment.GetBuildVars() # Bind our current execution environment into the shell vars. ph = os.path.dirname(sys.executable) if " " in ph: ph = '"' + ph + '"' shell_env.set_shell_var("PYTHON_HOME", ph) # PYTHON_COMMAND is required to be set for using edk2 python builds. # todo: work with edk2 to remove the bat file and move to native python calls. # This would be better in an edk2 plugin so that it could be modified/controlled # more easily # pc = sys.executable if " " in pc: pc = '"' + pc + '"' shell_env.set_shell_var("PYTHON_COMMAND", pc) env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list), "from edk2 ci build.py") # Generate consumable XML object- junit format JunitReport = JunitTestReport() # Keep track of failures failure_num = 0 total_num = 0 # Load plugins logging.log(edk2_logging.SECTION, "Loading plugins") pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin) for pkgToRunOn in self.requested_package_list: # # run all loaded Edk2CiBuild Plugins/Tests # logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package") logging.info(f"Running on Package: {pkgToRunOn}") package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}" ts = JunitReport.create_new_testsuite(pkgToRunOn, package_class_name) packagebuildlog_path = os.path.join(log_directory, pkgToRunOn) _, txt_handle = edk2_logging.setup_txt_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) _, md_handle = edk2_logging.setup_markdown_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) loghandle = [txt_handle, md_handle] shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() # load the package level .ci.yaml pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath( os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml")) if (pkg_config_file): with open(pkg_config_file, 'r') as f: pkg_config = yaml.safe_load(f) else: logging.info(f"No Pkg Config file for {pkgToRunOn}") pkg_config = dict() # get all the defines from the package configuration if "Defines" in pkg_config: for definition_key in pkg_config["Defines"]: definition = pkg_config["Defines"][definition_key] env.SetValue(definition_key, definition, "Edk2CiBuild.py from PkgConfig yaml", False) # For each plugin for Descriptor in pluginList: # For each target for target in self.requested_target_list: if (target not in Descriptor.Obj.RunsOnTargetList()): continue edk2_logging.log_progress( f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --" ) total_num += 1 shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() env.SetValue("TARGET", target, "Edk2CiBuild.py before RunBuildPlugin") (testcasename, testclassname) = Descriptor.Obj.GetTestName( package_class_name, env) tc = ts.create_new_testcase(testcasename, testclassname) # create the stream for the build log plugin_output_stream = edk2_logging.create_output_stream() # merge the repo level and package level for this specific plugin pkg_plugin_configuration = self.merge_config( self.PlatformSettings.GetPluginSettings(), pkg_config, Descriptor.descriptor) # Still need to see if the package decided this should be skipped if pkg_plugin_configuration is None or\ "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]: tc.SetSkipped() edk2_logging.log_progress( "--->Test Skipped by package! %s" % Descriptor.Name) else: try: # - package is the edk2 path to package. This means workspace/package path relative. # - edk2path object configured with workspace and packages path # - any additional command line args # - RepoConfig Object (dict) for the build # - PkgConfig Object (dict) # - EnvConfig Object # - Plugin Manager Instance # - Plugin Helper Obj Instance # - testcase Object used for outputing junit results # - output_stream the StringIO output stream from this plugin rc = Descriptor.Obj.RunBuildPlugin( pkgToRunOn, edk2path, pkg_plugin_configuration, env, self.plugin_manager, self.helper, tc, plugin_output_stream) except Exception as exp: exc_type, exc_value, exc_traceback = sys.exc_info() logging.critical("EXCEPTION: {0}".format(exp)) exceptionPrint = traceback.format_exception( type(exp), exp, exc_traceback) logging.critical(" ".join(exceptionPrint)) tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION") rc = 1 if (rc > 0): failure_num += 1 if (rc is None): logging.error( f"--->Test Failed: {Descriptor.Name} {target} returned NoneType" ) else: logging.error( f"--->Test Failed: {Descriptor.Name} {target} returned {rc}" ) elif (rc < 0): logging.warn( f"--->Test Skipped: in plugin! {Descriptor.Name} {target}" ) else: edk2_logging.log_progress( f"--->Test Success: {Descriptor.Name} {target}" ) # revert to the checkpoint we created previously shell_environment.RevertBuildVars() # remove the logger edk2_logging.remove_output_stream(plugin_output_stream) # finished target loop # Finished plugin loop edk2_logging.stop_logging( loghandle) # stop the logging for this particular buildfile shell_environment.RevertBuildVars() # Finished buildable file loop JunitReport.Output( os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml")) # Print Overall Success if (failure_num != 0): logging.error("Overall Build Status: Error") edk2_logging.log_progress( f"There were {failure_num} failures out of {total_num} attempts" ) else: edk2_logging.log_progress("Overall Build Status: Success") return failure_num
def do_post_build(self, thebuilder): starttime = datetime.now() logging.info( "---------------------------------------------------------") logging.info( "-----------Postbuild Image Validation Starting-----------") logging.info( "---------------------------------------------------------") # Load Configuration Data config_path = thebuilder.env.GetValue("PE_VALIDATION_PATH", None) tool_chain_tag = thebuilder.env.GetValue("TOOL_CHAIN_TAG") if config_path is None: logging.info( "PE_VALIDATION_PATH not set, PE Image Validation Skipped") return 0 # Path not set, Plugin skipped if not os.path.isfile(config_path): logging.error("Invalid PE_VALIDATION_PATH. File not Found") return 1 with open(config_path) as jsonfile: config_data = json.load(jsonfile) self.test_manager.config_data = config_data self.config_data = config_data self.ignore_list = config_data["IGNORE_LIST"] self.arch_dict = config_data["TARGET_ARCH"] count = 0 # Start Pre-Compiled Image Verification fdf_parser = FdfParser() dsc_parser = DscParser() ws = thebuilder.ws pp = thebuilder.pp.split(os.pathsep) edk2 = Edk2Path(ws, pp) ActiveDsc = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( thebuilder.env.GetValue("ACTIVE_PLATFORM")) ActiveFdf = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( thebuilder.env.GetValue("FLASH_DEFINITION")) if ActiveFdf is None: logging.info("No FDF found - PE Image Validation skipped") return 0 # parse the DSC and the FDF dsc_parser.SetBaseAbsPath(ws).SetPackagePaths(pp) dsc_parser.SetInputVars( thebuilder.env.GetAllBuildKeyValues()).ParseFile( ActiveDsc) # parse the DSC for build vars fdf_parser.SetBaseAbsPath(ws).SetPackagePaths(pp) fdf_parser.SetInputVars(dsc_parser.LocalVars).ParseFile( ActiveFdf) # give FDF parser the vars from DSC # Test all pre-compiled efis described in the fdf result = Result.PASS for FV_name in fdf_parser.FVs: # Get all Firmware volumes FV_files = fdf_parser.FVs[FV_name]["Files"] for fv_file_name in FV_files: # Iterate over each file in the firmware volume fv_file = FV_files[fv_file_name] if "PE32" in fv_file: # Any PE32 section in the FV contains a path to the efi # could have multiple PE32 sections for efi_path in fv_file["PE32"]: efi_path = self._resolve_vars(thebuilder, efi_path) efi_path = edk2.GetAbsolutePathOnThisSystemFromEdk2RelativePath( efi_path) if efi_path == None: logging.warn( "Unable to parse the path to the pre-compiled efi" ) continue if os.path.basename(efi_path) in self.ignore_list: continue logging.info( f'Performing Image Verification ... {os.path.basename(efi_path)}' ) if self._validate_image( efi_path, fv_file["type"]) == Result.FAIL: result = Result.FAIL count += 1 # End Pre-Compiled Image Verification # Start Build Time Compiled Image Verification result = Result.PASS for arch in thebuilder.env.GetValue("TARGET_ARCH").split(): efi_path_list = self._walk_directory_for_extension( ['.efi'], f'{thebuilder.env.GetValue("BUILD_OUTPUT_BASE")}/{arch}') for efi_path in efi_path_list: if os.path.basename(efi_path) in self.ignore_list: continue # Perform Image Verification on any output efi's # Grab profile from makefile if efi_path.__contains__("OUTPUT"): try: if tool_chain_tag.__contains__("VS"): profile = self._get_profile_from_makefile( f'{Path(efi_path).parent.parent}/Makefile') elif tool_chain_tag.__contains__("GCC"): profile = self._get_profile_from_makefile( f'{Path(efi_path).parent.parent}/GNUmakefile') elif tool_chain_tag.__contains__("CLANG"): profile = self._get_profile_from_makefile( f'{Path(efi_path).parent.parent}/GNUmakefile') else: logging.warn( "Unexpected TOOL_CHAIN_TAG... Cannot parse makefile. Using DEFAULT profile." ) profile = "DEFAULT" except: logging.warn( f'Failed to parse makefile at [{Path(efi_path).parent.parent}/GNUmakefile]' ) logging.warn(f'Using DEFAULT profile') profile = "DEFAULT" logging.info( f'Performing Image Verification ... {os.path.basename(efi_path)}' ) if self._validate_image(efi_path, profile) == Result.FAIL: result = Result.FAIL count += 1 # End Built Time Compiled Image Verification endtime = datetime.now() delta = endtime - starttime logging.info( "---------------------------------------------------------") logging.info( "-----------Postbuild Image Validation Finished-----------") logging.info( "------------------{:04d} Images Verified-------------------". format(count)) logging.info( "-------------- Running Time (mm:ss): {0[0]:02}:{0[1]:02} --------------" .format(divmod(delta.seconds, 60))) logging.info( "---------------------------------------------------------") if result == Result.FAIL: return 1 else: return 0
################################################ # This plugin python file is also # a command line tool # ################################################ if __name__ == '__main__': from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser from edk2toollib.uefi.edk2.path_utilities import Edk2Path # Parse required paths passed from cmd line arguments Paths = path_parse() dummy_list = [] pathtool = Edk2Path(Paths.WorkSpace, dummy_list) # Use absolute module path to find package path pkg_path = pathtool.GetContainingPackage(Paths.ModulePath) rel_path = Paths.ModulePath[Paths.ModulePath.find(pkg_path):] rel_path = rel_path.replace('\\', '/') mod_hash = ModuleHashCal(Paths.ModulePath) VERSION_INDEX = Paths.Version - 1 if VERSION_INDEX == 0: print( "Copy and paste the following line(s) to your overrider inf file(s):\n" ) print('#Override : %08d | %s | %s | %s' %
def test_nonexistant_ws(self): ''' test edk2path with invalid workspace''' invalid_ws = os.path.join(self.tmp, "invalidpath") with self.assertRaises(Exception): Edk2Path(invalid_ws, [])
def test_get_containing_package_outside_workspace(self): ''' test basic usage of GetContainingPackage with packages path outside the workspace File layout: root/ <-- current working directory (self.tmp) folder_ws/ <-- workspace root WSTestPkg/ <-- A edk2 package WSTestPkg.dec module1/ module1.inf module2/ module2.inf X64/ TestFile.c folder_pp/ <-- packages path PPTestPkg/ <-- A edk2 package PPTestPkg.DEC module1/ module1.INF module2/ module2.INF X64/ TestFile.c ''' ws_rel = "folder_ws" ws_abs = os.path.join(self.tmp, ws_rel) os.mkdir(ws_abs) folder_pp_rel = "pp1" folder_pp1_abs = os.path.join(self.tmp, folder_pp_rel) os.mkdir(folder_pp1_abs) ws_p_name = "WSTestPkg" ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name) pp_p_name = "PPTestPkg" pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False) pathobj = Edk2Path(ws_abs, [folder_pp1_abs]) # file in WSTestPkg root p = os.path.join(ws_pkg_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name) # file in module in WSTestPkg p = os.path.join(ws_pkg_abs, "module1", "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name) # file in workspace root - no package- should return ws root p = os.path.join(ws_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), ws_rel) # file outside of the workspace - invalid and should return None p = os.path.join(os.path.dirname(ws_abs), "testfile.c") self.assertIsNone(pathobj.GetContainingPackage(p)) # file in PPTestPkg root p = os.path.join(pp_pkg_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name) # file in module in WSTestPkg p = os.path.join(pp_pkg_abs, "module1", "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name) # file in packages path root - no package- should return packages path dir p = os.path.join(folder_pp1_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), folder_pp_rel)
def test_basic_init_ws_abs(self): ''' test edk2path with valid absolute path to workspace''' pathobj = Edk2Path(self.tmp, []) self.assertEqual(pathobj.WorkspacePath, self.tmp)
def test_get_containing_package_ws_abs_different_case(self): ''' test basic usage of GetContainingPackage when the workspace path has different case for the drive letter then the incoming paths. This can happen on Windows if os.path.realpath is used. File layout: root/ <-- current working directory (self.tmp) folder_ws/ <-- workspace root folder_pp/ <-- packages path PPTestPkg/ <-- A edk2 package PPTestPkg.DEC module1/ module1.INF module2/ module2.INF X64/ TestFile.c WSTestPkg/ <-- A edk2 package WSTestPkg.dec module1/ module1.inf module2/ module2.inf X64/ TestFile.c ''' ws_rel = "folder_ws" ws_abs = os.path.join(self.tmp, ws_rel) wsi_abs = os.path.join(self.tmp, ws_rel.capitalize()) # invert the case of the first char of the ws folder name os.mkdir(ws_abs) folder_pp_rel = "pp1" folder_pp1_abs = os.path.join(ws_abs, folder_pp_rel) os.mkdir(folder_pp1_abs) ws_p_name = "WSTestPkg" ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name) pp_p_name = "PPTestPkg" pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False) pathobj = Edk2Path(wsi_abs, [folder_pp1_abs]) # confirm inverted self.assertNotEqual(pathobj.WorkspacePath, ws_abs) self.assertEqual(os.path.normcase(pathobj.WorkspacePath), os.path.normcase(ws_abs)) # file in WSTestPkg root p = os.path.join(ws_pkg_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name) # file in module in WSTestPkg p = os.path.join(ws_pkg_abs, "module1", "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), ws_p_name) # file in workspace root - no package- should return ws root p = os.path.join(ws_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), ws_rel) # file outside of the workspace - invalid and should return None p = os.path.join(os.path.dirname(ws_abs), "testfile.c") self.assertIsNone(pathobj.GetContainingPackage(p)) # file in PPTestPkg root p = os.path.join(pp_pkg_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name) # file in module in WSTestPkg p = os.path.join(pp_pkg_abs, "module1", "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), pp_p_name) # file in packages path root - no package- should return packages path dir p = os.path.join(folder_pp1_abs, "testfile.c") self.assertEqual(pathobj.GetContainingPackage(p), folder_pp_rel)
def test_get_containing_module(self): ''' test basic usage of GetContainingModule with packages path nested inside the workspace File layout: root/ <-- current working directory (self.tmp) folder_ws/ <-- workspace root folder_pp/ <-- packages path PPTestPkg/ <-- A edk2 package PPTestPkg.DEC module1/ module1.INF module2/ module2.INF X64/ TestFile.c WSTestPkg/ <-- A edk2 package WSTestPkg.dec module1/ module1.inf module2/ module2.inf X64/ TestFile.c ''' ws_rel = "folder_ws" ws_abs = os.path.join(self.tmp, ws_rel) os.mkdir(ws_abs) folder_pp_rel = "pp1" folder_pp1_abs = os.path.join(ws_abs, folder_pp_rel) os.mkdir(folder_pp1_abs) ws_p_name = "WSTestPkg" ws_pkg_abs = self._make_edk2_package_helper(ws_abs, ws_p_name) pp_p_name = "PPTestPkg" pp_pkg_abs = self._make_edk2_package_helper(folder_pp1_abs, pp_p_name, extension_case_lower=False) pathobj = Edk2Path(ws_abs, [folder_pp1_abs]) # file in WSTestPkg root. Module list is empty p = os.path.join(ws_pkg_abs, "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file in module in WSTestPkg/module1/module1.inf p = os.path.join(ws_pkg_abs, "module1", "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 1) self.assertIn(os.path.join(ws_pkg_abs, "module1", "module1.inf"), relist) # file in workspace root - no package- should return ws root p = os.path.join(ws_abs, "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file outside of the workspace - invalid and should return None p = os.path.join(os.path.dirname(ws_abs), "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file in module2 x64 p = os.path.join(ws_pkg_abs, "module2", "X64", "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 1) self.assertIn(os.path.join(ws_pkg_abs, "module2", "module2.inf"), relist) # inf file in module2 x64 p = os.path.join(ws_pkg_abs, "module2", "module2.inf") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 1) self.assertIn(os.path.join(ws_pkg_abs, "module2", "module2.inf"), relist) # file in PPTestPkg root p = os.path.join(pp_pkg_abs, "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file in module in PPTestPkg p = os.path.join(pp_pkg_abs, "module1", "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 1) self.assertIn(os.path.join(pp_pkg_abs, "module1", "module1.INF"), relist) # inf file in module in PPTestPkg p = os.path.join(pp_pkg_abs, "module1", "module1.INF") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 1) self.assertIn(os.path.join(pp_pkg_abs, "module1", "module1.INF"), relist) # file in packages path root - no module p = os.path.join(folder_pp1_abs, "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file doesn't exist and parent folder doesn't exist p = os.path.join(ws_pkg_abs, "ThisParentDirDoesNotExist", "ThisFileDoesNotExist.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file doesn't exist and parent folder doesn't exist and parent parent folder doesn't exist p = os.path.join(ws_pkg_abs, "DirDirDoesNotExist", "DirDoesNotExist", "FileDoesNotExist.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 0) # file doesn't exist and parent folder doesn't exist but parent parent is valid module # file in module in WSTestPkg/module1/module1.inf p = os.path.join(ws_pkg_abs, "module1", "ThisParentDirDoesNotExist", "testfile.c") relist = pathobj.GetContainingModules(p) self.assertEqual(len(relist), 1) self.assertIn(os.path.join(ws_pkg_abs, "module1", "module1.inf"), relist)