Exemplo n.º 1
0
 def PerformUpdate(self):
     (build_env,
      shell_env) = self_describing_environment.BootstrapEnvironment(
          self.GetWorkspaceRoot(), self.GetActiveScopes())
     self_describing_environment.UpdateDependencies(self.GetWorkspaceRoot(),
                                                    self.GetActiveScopes())
     return (build_env, shell_env)
 def test_collect_path_env(self):
     ''' makes sure the SDE can collect path env '''
     scopes = ("global", )
     tree = uefi_tree(self.workspace, create_platform=False)
     tree.create_path_env("testing_corebuild",
                          var_name="hey",
                          flags=[
                              "set_path",
                          ])
     tree.create_path_env("testing_corebuild2",
                          var_name="hey",
                          flags=[
                              "set_pypath",
                          ])
     tree.create_path_env("testing_corebuild3",
                          var_name="hey",
                          flags=[
                              "set_build_var",
                          ])
     tree.create_path_env("testing_corebuild4",
                          var_name="hey",
                          flags=[
                              "set_shell_var",
                          ])
     build_env, shell_env = self_describing_environment.BootstrapEnvironment(
         self.workspace, scopes)
     self.assertEqual(len(build_env.paths), 4)
Exemplo n.º 3
0
 def PerformUpdate(self):
     ws_root = self.GetWorkspaceRoot()
     scopes = self.GetActiveScopes()
     (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(ws_root, scopes)
     (success, failure) = self_describing_environment.UpdateDependencies(ws_root, scopes)
     if success != 0:
         logging.log(edk2_logging.SECTION, f"\tUpdated/Verified {success} dependencies")
     return (build_env, shell_env, failure)
 def test_collect_path_env_scoped(self):
     ''' makes sure the SDE can collect path env with the right scopes '''
     scopes = ("global", "testing")
     tree = uefi_tree(self.workspace, create_platform=False)
     tree.create_path_env("testing_corebuild", scope="testing")
     tree.create_path_env("testing_corebuild2", scope="not_valid")
     build_env, shell_env = self_describing_environment.BootstrapEnvironment(
         self.workspace, scopes)
     self.assertEqual(len(build_env.paths), 1)
Exemplo n.º 5
0
 def test_duplicate_id_path_env(self):
     ''' check that the SDE will throw an exception if path_env have duplicate id's '''
     custom_scope = "global"
     scopes = (custom_scope,)
     tree = uefi_tree(self.workspace, create_platform=False)
     tree.create_path_env("testing_corebuild", dir_path="test1")
     tree.create_path_env("testing_corebuild")
     with self.assertRaises(RuntimeError):
         self_describing_environment.BootstrapEnvironment(self.workspace, scopes)
Exemplo n.º 6
0
 def test_override_path_env(self):
     ''' checks the SDE descriptor override system '''
     custom_scope = "global"
     scopes = (custom_scope,)
     tree = uefi_tree(self.workspace, create_platform=False)
     tree.create_path_env("testing_corebuild", var_name="hey", dir_path="test1", scope=custom_scope)
     tree.create_path_env("testing_corebuild2", var_name="jokes", scope=custom_scope,
                          extra_data={"override_id": "testing_corebuild"})
     build_env, shell_env = self_describing_environment.BootstrapEnvironment(self.workspace, scopes)
     print(build_env.paths)
     print(tree.get_workspace())
     self.assertEqual(len(build_env.paths), 1)
Exemplo n.º 7
0
    def Invoke(self):
        ''' Main process function.  Should not need to be overwritten '''

        self.ParseCommandLineOptions()
        self.ConfigureLogging()
        self.InputParametersConfiguredCallback()

        logging.log(edk2_logging.SECTION, "Init SDE")

        #
        # Next, get the environment set up.
        #
        (build_env,
         shell_env) = self_describing_environment.BootstrapEnvironment(
             self.GetWorkspaceRoot(), self.GetActiveScopes())

        # Make sure the environment verifies IF it is required for this invocation
        if self.GetVerifyCheckRequired(
        ) and not self_describing_environment.VerifyEnvironment(
                self.GetWorkspaceRoot(), self.GetActiveScopes()):
            raise RuntimeError(
                "SDE is not current.  Please update your env before running this tool."
            )

        # Load plugins
        logging.log(edk2_logging.SECTION, "Loading Plugins")

        self.plugin_manager = plugin_manager.PluginManager()
        failedPlugins = self.plugin_manager.SetListOfEnvironmentDescriptors(
            build_env.plugins)
        if failedPlugins:
            logging.critical(
                "One or more plugins failed to load. Halting build.")
            for a in failedPlugins:
                logging.error("Failed Plugin: {0}".format(a["name"]))
            raise Exception("One or more plugins failed to load.")

        self.helper = HelperFunctions()
        if (self.helper.LoadFromPluginManager(self.plugin_manager) > 0):
            raise Exception("One or more helper plugins failed to load.")

        logging.log(edk2_logging.SECTION, "Start Invocable Tool")
        retcode = self.Go()
        logging.log(edk2_logging.SECTION, "Summary")
        if (retcode != 0):
            edk2_logging.log_progress("Error")
        else:
            edk2_logging.log_progress("Success")

        logging.shutdown()
        sys.exit(retcode)
 def test_override_path_env_swapped_order(self):
     ''' checks the SDE descriptor override system with reversed paths so they are discovered in opposite order'''
     custom_scope = "global"
     scopes = (custom_scope, )
     tree = uefi_tree(self.workspace, create_platform=False)
     tree.create_path_env("testing_corebuild",
                          var_name="hey",
                          scope=custom_scope)
     tree.create_path_env(var_name="jokes",
                          dir_path="test1",
                          scope=custom_scope,
                          extra_data={"override_id": "testing_corebuild"})
     build_env, shell_env = self_describing_environment.BootstrapEnvironment(
         self.workspace, scopes)
     self.assertEqual(len(build_env.paths), 1)
Exemplo n.º 9
0
    def Go(self):
        logging.info("Running Python version: " + str(sys.version_info))

        Edk2PlatformBuild.collect_python_pip_info()

        (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
            self.GetWorkspaceRoot(), self.GetActiveScopes())

        # Bind our current execution environment into the shell vars.
        ph = os.path.dirname(sys.executable)
        if " " in ph:
            ph = '"' + ph + '"'
        shell_env.set_shell_var("PYTHON_HOME", ph)
        # PYTHON_COMMAND is required to be set for using edk2 python builds.
        # todo: work with edk2 to remove the bat file and move to native python calls
        pc = sys.executable
        if " " in pc:
            pc = '"' + pc + '"'
        shell_env.set_shell_var("PYTHON_COMMAND", pc)

        # Load plugins
        logging.log(edk2_logging.SECTION, "Loading Plugins")
        pm = plugin_manager.PluginManager()
        failedPlugins = pm.SetListOfEnvironmentDescriptors(
            build_env.plugins)
        if failedPlugins:
            logging.critical("One or more plugins failed to load. Halting build.")
            for a in failedPlugins:
                logging.error("Failed Plugin: {0}".format(a["name"]))
            raise Exception("One or more plugins failed to load.")

        helper = HelperFunctions()
        if(helper.LoadFromPluginManager(pm) > 0):
            raise Exception("One or more helper plugins failed to load.")

        # Make a pathobj so we can normalize and validate the workspace
        # and packages path.  The Settings Manager can return absolute or
        # relative paths
        pathobj = Edk2Path(self.GetWorkspaceRoot(), self.GetPackagesPath())
        #
        # Now we can actually kick off a build.
        #
        logging.log(edk2_logging.SECTION, "Kicking off build")
        ret = self.PlatformBuilder.Go(pathobj.WorkspacePath,
                                      os.pathsep.join(pathobj.PackagePathList),
                                      helper, pm)
        logging.log(edk2_logging.SECTION, f"Log file is located at: {self.log_filename}")
        return ret
    def Go(self):
        logging.info("Running Python version: " + str(sys.version_info))

        display_pip_package_info(PIP_PACKAGES_LIST)

        (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
            self.GetWorkspaceRoot(), self.GetActiveScopes())

        # Bind our current execution environment into the shell vars.
        ph = os.path.dirname(sys.executable)
        if " " in ph:
            ph = '"' + ph + '"'
        shell_env.set_shell_var("PYTHON_HOME", ph)
        # PYTHON_COMMAND is required to be set for using edk2 python builds.
        # todo: work with edk2 to remove the bat file and move to native python calls
        pc = sys.executable
        if " " in pc:
            pc = '"' + pc + '"'
        shell_env.set_shell_var("PYTHON_COMMAND", pc)

        # Load plugins
        logging.log(edk2_logging.SECTION, "Loading Plugins")
        pm = plugin_manager.PluginManager()
        failedPlugins = pm.SetListOfEnvironmentDescriptors(
            build_env.plugins)
        if failedPlugins:
            logging.critical("One or more plugins failed to load. Halting build.")
            for a in failedPlugins:
                logging.error("Failed Plugin: {0}".format(a["name"]))
            raise Exception("One or more plugins failed to load.")

        helper = HelperFunctions()
        if(helper.LoadFromPluginManager(pm) > 0):
            raise Exception("One or more helper plugins failed to load.")
        #
        # Now we can actually kick off a build.
        #
        logging.log(edk2_logging.SECTION, "Kicking off build")
        return self.PlatformBuilder.Go(self.GetWorkspaceRoot(),
                                       self.PlatformSettings.GetModulePkgsPath(),
                                       helper, pm)
 def test_multiple_override_path_env(self):
     ''' checks the SDE descriptor override system will throw an error on multiple overrides'''
     custom_scope = "global"
     scopes = (custom_scope, )
     tree = uefi_tree(self.workspace, create_platform=False)
     tree.create_path_env("testing_corebuild",
                          var_name="hey",
                          dir_path="test1",
                          scope=custom_scope)
     tree.create_path_env("testing_corebuild2",
                          var_name="jokes",
                          scope=custom_scope,
                          extra_data={"override_id": "testing_corebuild"})
     tree.create_path_env("testing_corebuild3",
                          var_name="laughs",
                          scope=custom_scope,
                          extra_data={"override_id": "testing_corebuild"})
     # we should get an exception because we have two overrides
     with self.assertRaises(RuntimeError):
         build_env, shell_env = self_describing_environment.BootstrapEnvironment(
             self.workspace, scopes)
         self.fail()
Exemplo n.º 12
0
    def Go(self):
        logging.info("Running Python version: " + str(sys.version_info))

        (build_env,
         shell_env) = self_describing_environment.BootstrapEnvironment(
             self.GetWorkspaceRoot(), self.GetActiveScopes())

        # # Bind our current execution environment into the shell vars.
        ph = os.path.dirname(sys.executable)
        if " " in ph:
            ph = '"' + ph + '"'
        shell_env.set_shell_var("PYTHON_HOME", ph)
        # PYTHON_COMMAND is required to be set for using edk2 python builds.
        pc = sys.executable
        if " " in pc:
            pc = '"' + pc + '"'
        shell_env.set_shell_var("PYTHON_COMMAND", pc)

        if self.tool_chain_tag.lower().startswith("vs"):

            # # Update environment with required VC vars.
            interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
            interesting_keys.extend([
                "LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion",
                "WindowsLibPath", "WindowsSdkBinPath"
            ])
            interesting_keys.extend([
                "WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion",
                "VCToolsInstallDir"
            ])
            vc_vars = QueryVcVariables(interesting_keys,
                                       'x86',
                                       vs_version=self.tool_chain_tag.lower())
            for key in vc_vars.keys():
                logging.debug(f"Var - {key} = {vc_vars[key]}")
                if key.lower() == 'path':
                    shell_env.insert_path(vc_vars[key])
                else:
                    shell_env.set_shell_var(key, vc_vars[key])

            self.OutputDir = os.path.join(
                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")

            # compiled tools need to be added to path because antlr is referenced
            shell_env.insert_path(self.OutputDir)

            # Actually build the tools.
            ret = RunCmd('nmake.exe',
                         None,
                         workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
            if ret != 0:
                raise Exception("Failed to build.")

            self.WritePathEnvFile(self.OutputDir)
            return ret

        elif self.tool_chain_tag.lower().startswith("gcc"):
            ret = RunCmd("make",
                         "-C .",
                         workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
            if ret != 0:
                raise Exception("Failed to build.")

            self.OutputDir = os.path.join(
                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C",
                "bin")

            self.WritePathEnvFile(self.OutputDir)
            return ret

        logging.critical("Tool Chain not supported")
        return -1
    def Go(self):
        log_directory = os.path.join(self.GetWorkspaceRoot(),
                                     self.GetLoggingFolderRelativeToRoot())

        # SET PACKAGE PATH
        #
        # Get Package Path from config file
        pplist = self.PlatformSettings.GetPackagesPath(
        ) if self.PlatformSettings.GetPackagesPath() else []

        # Check Dependencies for Repo
        for dependency in self.PlatformSettings.GetDependencies():
            pplist.append(dependency["Path"])

        # make Edk2Path object to handle all path operations
        try:
            edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
        except Exception as e:
            logging.error(
                "You need to run stewart_ci_setup to resolve all repos.")
            raise e

        logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
        logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
        logging.info(
            f"Package Path: {self.PlatformSettings.GetPackagesPath()}")
        # logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version))
        # logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
        # logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
        # Bring up the common minimum environment.
        logging.log(edk2_logging.SECTION, "Getting Enviroment")
        (build_env,
         shell_env) = self_describing_environment.BootstrapEnvironment(
             self.GetWorkspaceRoot(), self.GetActiveScopes())
        env = shell_environment.GetBuildVars()

        # Bind our current execution environment into the shell vars.
        ph = os.path.dirname(sys.executable)
        if " " in ph:
            ph = '"' + ph + '"'
        shell_env.set_shell_var("PYTHON_HOME", ph)
        # PYTHON_COMMAND is required to be set for using edk2 python builds.
        # todo: work with edk2 to remove the bat file and move to native python calls
        pc = sys.executable
        if " " in pc:
            pc = '"' + pc + '"'
        shell_env.set_shell_var("PYTHON_COMMAND", pc)

        archSupported = " ".join(self.PlatformSettings.GetArchSupported())
        env.SetValue("TARGET_ARCH", archSupported,
                     "from PlatformSettings.GetArchSupported()")

        _targets = " ".join(self.PlatformSettings.GetTargetsSupported())

        # Generate consumable XML object- junit format
        JunitReport = JunitTestReport()

        # Keep track of failures
        failure_num = 0
        total_num = 0

        # Load plugins
        logging.log(edk2_logging.SECTION, "Loading plugins")

        pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
        if len(self.packageList) == 0:
            self.packageList.extend(self.PlatformSettings.GetPackages())

        for pkgToRunOn in self.packageList:
            #
            # run all loaded Edk2CiBuild Plugins/Tests
            #
            logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
            logging.info(f"Running on Package: {pkgToRunOn}")
            ts = JunitReport.create_new_testsuite(
                pkgToRunOn,
                f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}"
            )
            packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
            _, txthandle = edk2_logging.setup_txt_logger(
                packagebuildlog_path,
                f"BUILDLOG_{pkgToRunOn}",
                logging_level=logging.DEBUG,
                isVerbose=True)
            _, mdhandle = edk2_logging.setup_markdown_logger(
                packagebuildlog_path,
                f"BUILDLOG_{pkgToRunOn}",
                logging_level=logging.DEBUG,
                isVerbose=True)
            loghandle = [txthandle, mdhandle]
            shell_environment.CheckpointBuildVars()
            env = shell_environment.GetBuildVars()

            # load the package level .mu.json
            pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
                os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml"))
            if (pkg_config_file):
                with open(pkg_config_file, 'r') as f:
                    pkg_config = yaml.safe_load(f)
            else:
                logging.info(f"No Pkg Config file for {pkgToRunOn}")
                pkg_config = dict()

            # check the resulting configuration
            config_validator.check_package_confg(pkgToRunOn, pkg_config,
                                                 pluginList)

            # get all the defines from the package configuration
            if "Defines" in pkg_config:
                for definition_key in pkg_config["Defines"]:
                    definition = pkg_config["Defines"][definition_key]
                    env.SetValue(definition_key, definition,
                                 "Edk2CiBuild.py from PkgConfig yaml", False)

            for Descriptor in pluginList:
                # Get our targets
                targets = ["DEBUG"]
                if Descriptor.Obj.IsTargetDependent() and _targets:
                    targets = self.PlatformSettings.GetTargetsSupported()

                for target in targets:
                    edk2_logging.log_progress(
                        f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
                    )
                    total_num += 1
                    shell_environment.CheckpointBuildVars()
                    env = shell_environment.GetBuildVars()

                    env.SetValue("TARGET", target,
                                 "Edk2CiBuild.py before RunBuildPlugin")
                    (testcasename, testclassname) = Descriptor.Obj.GetTestName(
                        pkgToRunOn, env)
                    tc = ts.create_new_testcase(testcasename, testclassname)

                    # create the stream for the build log
                    plugin_output_stream = edk2_logging.create_output_stream()

                    # merge the repo level and package level for this specific plugin
                    pkg_plugin_configuration = merge_config(
                        self.PlatformSettings.GetPluginSettings(), pkg_config,
                        Descriptor.descriptor)

                    # perhaps we should ask the validator to run on the package for this target

                    # Still need to see if the package decided this should be skipped
                    if pkg_plugin_configuration is None or\
                            "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
                        tc.SetSkipped()
                        edk2_logging.log_progress(
                            "--->Test Skipped by package! %s" %
                            Descriptor.Name)

                    else:
                        try:
                            #   - package is the edk2 path to package.  This means workspace/packagepath relative.
                            #   - edk2path object configured with workspace and packages path
                            #   - any additional command line args
                            #   - RepoConfig Object (dict) for the build
                            #   - PkgConfig Object (dict)
                            #   - EnvConfig Object
                            #   - Plugin Manager Instance
                            #   - Plugin Helper Obj Instance
                            #   - testcase Object used for outputing junit results
                            #   - output_stream the StringIO output stream from this plugin
                            rc = Descriptor.Obj.RunBuildPlugin(
                                pkgToRunOn, edk2path, pkg_plugin_configuration,
                                env, self.plugin_manager, self.helper, tc,
                                plugin_output_stream)
                        except Exception as exp:
                            exc_type, exc_value, exc_traceback = sys.exc_info()
                            logging.critical("EXCEPTION: {0}".format(exp))
                            exceptionPrint = traceback.format_exception(
                                type(exp), exp, exc_traceback)
                            logging.critical(" ".join(exceptionPrint))
                            tc.SetError("Exception: {0}".format(exp),
                                        "UNEXPECTED EXCEPTION")
                            rc = 1

                        if (rc != 0):
                            failure_num += 1
                            if (rc is None):
                                logging.error(
                                    "--->Test Failed: %s returned NoneType" %
                                    Descriptor.Name)
                            else:
                                logging.error(
                                    "--->Test Failed: %s returned %d" %
                                    (Descriptor.Name, rc))
                        else:
                            edk2_logging.log_progress(
                                f"--->Test Success {Descriptor.Name} {target}")

                    # revert to the checkpoint we created previously
                    shell_environment.RevertBuildVars()
                    # remove the logger
                    edk2_logging.remove_output_stream(plugin_output_stream)
                # finished target loop
            # Finished plugin loop

            edk2_logging.stop_logging(
                loghandle)  # stop the logging for this particular buildfile
            shell_environment.RevertBuildVars()
        # Finished buildable file loop

        JunitReport.Output(
            os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))

        # Print Overall Success
        if (failure_num != 0):
            logging.error("Overall Build Status: Error")
            edk2_logging.log_progress(
                f"There were {failure_num} failures out of {total_num} attempts"
            )
        else:
            edk2_logging.log_progress("Overall Build Status: Success")

        return failure_num
    def Go(self):
        log_directory = os.path.join(self.GetWorkspaceRoot(),
                                     self.GetLoggingFolderRelativeToRoot())

        Edk2CiBuild.collect_python_pip_info()

        #
        # Get Package Path from config file
        pplist = self.PlatformSettings.GetPackagesPath(
        ) if self.PlatformSettings.GetPackagesPath() else []

        # make Edk2Path object to handle all path operations
        try:
            edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
        except Exception as e:
            logging.error("Src Tree is invalid.  Did you Setup correctly?")
            raise e

        logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
        logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
        logging.info(f"Package Path: {pplist}")
        # Bring up the common minimum environment.
        logging.log(edk2_logging.SECTION, "Getting Environment")
        (build_env,
         shell_env) = self_describing_environment.BootstrapEnvironment(
             self.GetWorkspaceRoot(), self.GetActiveScopes())
        env = shell_environment.GetBuildVars()

        # Bind our current execution environment into the shell vars.
        ph = os.path.dirname(sys.executable)
        if " " in ph:
            ph = '"' + ph + '"'
        shell_env.set_shell_var("PYTHON_HOME", ph)
        # PYTHON_COMMAND is required to be set for using edk2 python builds.
        # todo: work with edk2 to remove the bat file and move to native python calls.
        #       This would be better in an edk2 plugin so that it could be modified/controlled
        #       more easily
        #
        pc = sys.executable
        if " " in pc:
            pc = '"' + pc + '"'
        shell_env.set_shell_var("PYTHON_COMMAND", pc)

        env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list),
                     "from edk2 ci build.py")

        # Generate consumable XML object- junit format
        JunitReport = JunitTestReport()

        # Keep track of failures
        failure_num = 0
        total_num = 0

        # Load plugins
        logging.log(edk2_logging.SECTION, "Loading plugins")

        pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)

        for pkgToRunOn in self.requested_package_list:
            #
            # run all loaded Edk2CiBuild Plugins/Tests
            #
            logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
            logging.info(f"Running on Package: {pkgToRunOn}")
            package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}"
            ts = JunitReport.create_new_testsuite(pkgToRunOn,
                                                  package_class_name)
            packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
            _, txt_handle = edk2_logging.setup_txt_logger(
                packagebuildlog_path,
                f"BUILDLOG_{pkgToRunOn}",
                logging_level=logging.DEBUG,
                isVerbose=True)
            _, md_handle = edk2_logging.setup_markdown_logger(
                packagebuildlog_path,
                f"BUILDLOG_{pkgToRunOn}",
                logging_level=logging.DEBUG,
                isVerbose=True)
            loghandle = [txt_handle, md_handle]
            shell_environment.CheckpointBuildVars()
            env = shell_environment.GetBuildVars()

            # load the package level .ci.yaml
            pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
                os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml"))
            if (pkg_config_file):
                with open(pkg_config_file, 'r') as f:
                    pkg_config = yaml.safe_load(f)
            else:
                logging.info(f"No Pkg Config file for {pkgToRunOn}")
                pkg_config = dict()

            # get all the defines from the package configuration
            if "Defines" in pkg_config:
                for definition_key in pkg_config["Defines"]:
                    definition = pkg_config["Defines"][definition_key]
                    env.SetValue(definition_key, definition,
                                 "Edk2CiBuild.py from PkgConfig yaml", False)

            # For each plugin
            for Descriptor in pluginList:
                # For each target
                for target in self.requested_target_list:

                    if (target not in Descriptor.Obj.RunsOnTargetList()):
                        continue

                    edk2_logging.log_progress(
                        f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
                    )
                    total_num += 1
                    shell_environment.CheckpointBuildVars()
                    env = shell_environment.GetBuildVars()

                    env.SetValue("TARGET", target,
                                 "Edk2CiBuild.py before RunBuildPlugin")
                    (testcasename, testclassname) = Descriptor.Obj.GetTestName(
                        package_class_name, env)
                    tc = ts.create_new_testcase(testcasename, testclassname)

                    # create the stream for the build log
                    plugin_output_stream = edk2_logging.create_output_stream()

                    # merge the repo level and package level for this specific plugin
                    pkg_plugin_configuration = self.merge_config(
                        self.PlatformSettings.GetPluginSettings(), pkg_config,
                        Descriptor.descriptor)

                    # Still need to see if the package decided this should be skipped
                    if pkg_plugin_configuration is None or\
                            "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
                        tc.SetSkipped()
                        edk2_logging.log_progress(
                            "--->Test Skipped by package! %s" %
                            Descriptor.Name)

                    else:
                        try:
                            #   - package is the edk2 path to package.  This means workspace/package path relative.
                            #   - edk2path object configured with workspace and packages path
                            #   - any additional command line args
                            #   - RepoConfig Object (dict) for the build
                            #   - PkgConfig Object (dict)
                            #   - EnvConfig Object
                            #   - Plugin Manager Instance
                            #   - Plugin Helper Obj Instance
                            #   - testcase Object used for outputing junit results
                            #   - output_stream the StringIO output stream from this plugin
                            rc = Descriptor.Obj.RunBuildPlugin(
                                pkgToRunOn, edk2path, pkg_plugin_configuration,
                                env, self.plugin_manager, self.helper, tc,
                                plugin_output_stream)
                        except Exception as exp:
                            exc_type, exc_value, exc_traceback = sys.exc_info()
                            logging.critical("EXCEPTION: {0}".format(exp))
                            exceptionPrint = traceback.format_exception(
                                type(exp), exp, exc_traceback)
                            logging.critical(" ".join(exceptionPrint))
                            tc.SetError("Exception: {0}".format(exp),
                                        "UNEXPECTED EXCEPTION")
                            rc = 1

                        if (rc > 0):
                            failure_num += 1
                            if (rc is None):
                                logging.error(
                                    f"--->Test Failed: {Descriptor.Name} {target} returned NoneType"
                                )
                            else:
                                logging.error(
                                    f"--->Test Failed: {Descriptor.Name} {target} returned {rc}"
                                )
                        elif (rc < 0):
                            logging.warn(
                                f"--->Test Skipped: in plugin! {Descriptor.Name} {target}"
                            )
                        else:
                            edk2_logging.log_progress(
                                f"--->Test Success: {Descriptor.Name} {target}"
                            )

                    # revert to the checkpoint we created previously
                    shell_environment.RevertBuildVars()
                    # remove the logger
                    edk2_logging.remove_output_stream(plugin_output_stream)
                # finished target loop
            # Finished plugin loop

            edk2_logging.stop_logging(
                loghandle)  # stop the logging for this particular buildfile
            shell_environment.RevertBuildVars()
        # Finished buildable file loop

        JunitReport.Output(
            os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))

        # Print Overall Success
        if (failure_num != 0):
            logging.error("Overall Build Status: Error")
            edk2_logging.log_progress(
                f"There were {failure_num} failures out of {total_num} attempts"
            )
        else:
            edk2_logging.log_progress("Overall Build Status: Success")

        return failure_num
def do_update(directory, scopes):
    (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
        directory, scopes)
    self_describing_environment.UpdateDependencies(directory, scopes)
    return (build_env, shell_env)