def RetrieveCommandLineOptions(self, args): ''' Retrieve command line options from the argparser ''' shell_environment.GetBuildVars().SetValue( "TARGET_ARCH", args.build_arch.upper(), "From CmdLine") shell_environment.GetBuildVars().SetValue( "ACTIVE_PLATFORM", os.path.join("VExpressPkg", "ArmVExpress-FVP-AArch64.dsc"), "From CmdLine")
def RetrieveCommandLineOptions(self, args): ''' Retrieve command line options from the argparser ''' shell_environment.GetBuildVars().SetValue("TARGET_ARCH", args.build_arch.upper(), "From CmdLine") shell_environment.GetBuildVars().SetValue( "ACTIVE_PLATFORM", "EmulatorPkg/EmulatorPkg.dsc", "From CmdLine")
def Go(self): ret = 0 env = shell_environment.GetBuildVars() env.SetValue("PRODUCT_NAME", self.PlatformSettings.GetName(), "Platform Hardcoded") env.SetValue("BLD_*_BUILDID_STRING", "201905", "Current Version") env.SetValue("BUILDREPORTING", "TRUE", "Platform Hardcoded") env.SetValue("BUILDREPORT_TYPES", 'PCD DEPEX LIBRARY BUILD_FLAGS', "Platform Hardcoded") # make sure python_command is set python_command = sys.executable if " " in python_command: python_command = '"' + python_command + '"' shell_environment.GetEnvironment().set_shell_var( "PYTHON_COMMAND", python_command) # Run pre build hook ret += self.PlatformSettings.PreFirstBuildHook() ws = self.GetWorkspaceRoot() pp = self.PlatformSettings.GetModulePkgsPath() # run each configuration ret = 0 try: for config in self.PlatformSettings.GetConfigurations(): pre_ret = self.PlatformSettings.PreBuildHook( ) # run pre build hook if pre_ret != 0: ret = pre_ret raise RuntimeError("We failed in prebuild hook") edk2_logging.log_progress(f"--Running next configuration--") logging.info(config) shell_environment.CheckpointBuildVars( ) # checkpoint our config env = shell_environment.GetBuildVars() # go through the config and apply to environement for key in config: env.SetValue(key, config[key], "provided by configuration") # make sure to set this after in case the config did env.SetValue("TOOL_CHAIN_TAG", "VS2017", "provided by builder") platformBuilder = UefiBuilder() # create our builder build_ret = platformBuilder.Go(ws, pp, self.helper, self.plugin_manager) # we always want to run the post build hook post_ret = self.PlatformSettings.PostBuildHook(ret) if build_ret != 0: ret = build_ret raise RuntimeError("We failed in build") if post_ret != 0: ret = post_ret raise RuntimeError("We failed in postbuild hook") shell_environment.RevertBuildVars() except RuntimeError: pass finally: # make sure to do our final build hook self.PlatformSettings.PostFinalBuildHook(ret) return ret
def RetrieveCommandLineOptions(self, args): ''' Retrieve command line options from the argparser ''' shell_environment.GetBuildVars().SetValue( "TARGET_ARCH", " ".join(args.build_arch.upper().split(",")), "From CmdLine") dsc = CommonPlatform.GetDscName(args.build_arch) shell_environment.GetBuildVars().SetValue("ACTIVE_PLATFORM", f"OvmfPkg/{dsc}", "From CmdLine")
def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' scopes = CommonPlatform.Scopes ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") Arch = shell_environment.GetBuildVars().GetValue("TARGET_ARCH", "") if GetHostInfo().os.upper() == "LINUX" and ActualToolChainTag.upper().startswith("GCC"): if "AARCH64" == Arch: scopes += ("gcc_aarch64_linux",) elif "ARM" == Arch: scopes += ("gcc_arm_linux",) return scopes
def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' if self.ActualScopes is None: scopes = ("cibuild", "edk2-build", "host-based-test") self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") is_linux = GetHostInfo().os.upper() == "LINUX" if self.UseBuiltInBaseTools is None: is_linux = GetHostInfo().os.upper() == "LINUX" # try and import the pip module for basetools try: import edk2basetools self.UseBuiltInBaseTools = True except ImportError: self.UseBuiltInBaseTools = False pass if self.UseBuiltInBaseTools == True: scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',) logging.warning("Using Pip Tools based BaseTools") else: logging.warning("Falling back to using in-tree BaseTools") if is_linux and self.ActualToolChainTag.upper().startswith("GCC"): if "AARCH64" in self.ActualArchitectures: scopes += ("gcc_aarch64_linux",) if "ARM" in self.ActualArchitectures: scopes += ("gcc_arm_linux",) if "RISCV64" in self.ActualArchitectures: scopes += ("gcc_riscv64_unknown",) self.ActualScopes = scopes return self.ActualScopes
def test_special_build_vars_should_be_checkpointable(self): shell_env = SE.ShellEnvironment() build_vars = SE.GetBuildVars() # This test is basically a rehash of the object checkpointing test, # but this time with the special vars. test_var1_name = 'SE_TEST_VAR_3' test_var1_data = 'MyData1' test_var1_data2 = 'RevisedData1' test_var1_data3 = 'MoreRevisedData1' test_var2_name = 'SE_TEST_VAR_4' test_var2_data = 'MyData2' # Set the first data and make a checkpoint. build_vars.SetValue(test_var1_name, test_var1_data, 'var1 set', overridable=True) check_point1 = shell_env.checkpoint() # Update previous value and set second data. Then checkpoint. build_vars.SetValue(test_var1_name, test_var1_data2, 'var1 set', overridable=True) build_vars.SetValue(test_var2_name, test_var2_data, 'var2 set', overridable=True) check_point2 = shell_env.checkpoint() # Restore the first checkpoint and verify values. shell_env.restore_checkpoint(check_point1) self.assertEqual(shell_env.get_build_var(test_var1_name), test_var1_data) self.assertIs(shell_env.get_build_var(test_var2_name), None) # Make a change to be tested later. build_vars.SetValue(test_var1_name, test_var1_data3, 'var1 set', overridable=True) self.assertEqual( shell_env.get_build_var(test_var1_name), test_var1_data3, 'even after restore, special build vars should always update current' ) # Restore the second checkpoint and verify values. shell_env.restore_checkpoint(check_point2) self.assertEqual(shell_env.get_build_var(test_var1_name), test_var1_data2) self.assertEqual(shell_env.get_build_var(test_var2_name), test_var2_data) # Restore the first checkpoint again and make sure original value still stands. shell_env.restore_checkpoint(check_point1) self.assertEqual(shell_env.get_build_var(test_var1_name), test_var1_data)
def test_special_build_vars_should_always_update_current(self): shell_env = SE.ShellEnvironment() build_vars = SE.GetBuildVars() test_var1_name = 'SE_TEST_VAR_update_current1' test_var1_data = 'NewData1' test_var1_data2 = 'NewerData1' test_var2_name = 'SE_TEST_VAR_update_current2' test_var2_data = 'NewData2' # Make a change and checkpoint. build_vars.SetValue(test_var1_name, test_var1_data, 'var1 set', overridable=True) shell_env.checkpoint() # Make a couple more changes. build_vars.SetValue(test_var1_name, test_var1_data2, 'var1 set', overridable=True) build_vars.SetValue(test_var2_name, test_var2_data, 'var2 set', overridable=True) # Make sure that the newer changes are valid. self.assertEqual(shell_env.get_build_var(test_var1_name), test_var1_data2) self.assertEqual(shell_env.get_build_var(test_var2_name), test_var2_data)
def GetPackagesPath(self): ''' Return a list of workspace relative paths that should be mapped as edk2 PackagesPath ''' result = [ shell_environment.GetBuildVars().GetValue("FEATURE_CONFIG_PATH", "") ] for a in CommonPlatform.PackagesPath: result.append(a) return result
def __init__(self, OverrideConf, AdditionalTemplateConfDir): self.Logger = logging.getLogger("ConfMgmt") self.env = shell_environment.GetBuildVars() if (self.env.GetValue("WORKSPACE") is None) or \ (self.env.GetValue("EDK2_BASE_TOOLS_DIR") is None): raise Exception( "WORKSPACE and EDK2_BASE_TOOLS_DIR must be set prior to running ConfMgmt" ) self.__PopulateConf(OverrideConf, AdditionalTemplateConfDir)
def GetName(self): ''' Get the name of the repo, platform, or product being build ''' ''' Used for naming the log file, among others ''' # check the startup nsh flag and if set then rename the log file. # this helps in CI so we don't overwrite the build log since running # uses the stuart_build command. if(shell_environment.GetBuildVars().GetValue("MAKE_STARTUP_NSH", "FALSE") == "TRUE"): return "OvmfPkg_With_Run" return "OvmfPkg"
def test_get_build_vars_should_update_vars(self): shell_env = SE.ShellEnvironment() build_vars = SE.GetBuildVars() test_var_name = 'SE_TEST_VAR_4' test_var_data = 'NewData1' build_vars.SetValue(test_var_name, test_var_data, 'random set') self.assertEqual(shell_env.get_build_var(test_var_name), test_var_data)
def RetrieveCommandLineOptions(self, args): ''' Retrieve command line options from the argparser ''' shell_environment.GetBuildVars().SetValue("TOOL_CHAIN_TAG", "VS2017", "Set default") if args.api_key is not None: self.api_key = args.api_key print("Using API KEY") self.nuget_version = args.nug_ver self.should_dump_version = args.dump_version
def test_go_skip_building(self): builder = uefi_build.UefiBuilder() builder.SkipPostBuild = True builder.SkipBuild = True builder.SkipBuild = True manager = PluginManager() shell_environment.GetBuildVars().SetValue("EDK_TOOLS_PATH", self.WORKSPACE, "empty") helper = uefi_helper_plugin.HelperFunctions() ret = builder.Go(self.WORKSPACE, "", helper, manager) self.assertEqual(ret, 0)
def GetPackagesPath(self): ''' Return a list of workspace relative paths that should be mapped as edk2-platforms PackagesPath ''' packages = [] edk2_repo = shell_environment.GetBuildVars().GetValue("EDK2_REPO", "") if not edk2_repo: edk2_repo = os.path.join("edk2_extdep", "edk2") packages.append(edk2_repo) packages.append(os.path.join("Platform", "ARM")) return packages
def test_special_build_vars_should_default_non_overrideable(self): shell_env = SE.ShellEnvironment() build_vars = SE.GetBuildVars() test_var_name = 'SE_TEST_VAR_4' test_var_data = 'NewData1' test_var_data2 = 'NewerData1' build_vars.SetValue(test_var_name, test_var_data, 'random set') build_vars.SetValue(test_var_name, test_var_data2, 'another random set') self.assertEqual(shell_env.get_build_var(test_var_name), test_var_data)
def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' scopes = ("cibuild","edk2-build") self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") if GetHostInfo().os.upper() == "LINUX" and self.ActualToolChainTag.upper().startswith("GCC"): if "AARCH64" in self.ActualArchitectures: scopes += ("gcc_aarch64_linux",) if "ARM" in self.ActualArchitectures: scopes += ("gcc_arm_linux",) return scopes
def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' scopes = ("corebuild", "project_mu","cibuild") self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") if (GetHostInfo().os == "Linux" and "AARCH64" in self.ActualArchitectures and self.ActualToolChainTag.upper().startswith("GCC")): scopes += ("gcc_aarch64_linux",) if GetHostInfo().os == "Windows": scopes += ("host-test-win",) return scopes
def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' if self.ActualScopes is None: scopes = ("cibuild", "edk2-build", "host-based-test") self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") is_linux = GetHostInfo().os.upper() == "LINUX" scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',) if is_linux and self.ActualToolChainTag.upper().startswith("GCC"): if "AARCH64" in self.ActualArchitectures: scopes += ("gcc_aarch64_linux",) if "ARM" in self.ActualArchitectures: scopes += ("gcc_arm_linux",) if "RISCV64" in self.ActualArchitectures: scopes += ("gcc_riscv64_unknown",) # If EDK2_REPO is not provided, download it. if not shell_environment.GetBuildVars().GetValue("EDK2_REPO", ""): scopes += ("edk2-repo",) self.ActualScopes = scopes return self.ActualScopes
def Go(self): log_directory = os.path.join(self.GetWorkspaceRoot(), self.GetLoggingFolderRelativeToRoot()) # SET PACKAGE PATH # # Get Package Path from config file pplist = self.PlatformSettings.GetPackagesPath( ) if self.PlatformSettings.GetPackagesPath() else [] # Check Dependencies for Repo for dependency in self.PlatformSettings.GetDependencies(): pplist.append(dependency["Path"]) # make Edk2Path object to handle all path operations try: edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist) except Exception as e: logging.error( "You need to run stewart_ci_setup to resolve all repos.") raise e logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}") logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}") logging.info( f"Package Path: {self.PlatformSettings.GetPackagesPath()}") # logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version)) # logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version) # logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version) # Bring up the common minimum environment. logging.log(edk2_logging.SECTION, "Getting Enviroment") (build_env, shell_env) = self_describing_environment.BootstrapEnvironment( self.GetWorkspaceRoot(), self.GetActiveScopes()) env = shell_environment.GetBuildVars() # Bind our current execution environment into the shell vars. ph = os.path.dirname(sys.executable) if " " in ph: ph = '"' + ph + '"' shell_env.set_shell_var("PYTHON_HOME", ph) # PYTHON_COMMAND is required to be set for using edk2 python builds. # todo: work with edk2 to remove the bat file and move to native python calls pc = sys.executable if " " in pc: pc = '"' + pc + '"' shell_env.set_shell_var("PYTHON_COMMAND", pc) archSupported = " ".join(self.PlatformSettings.GetArchSupported()) env.SetValue("TARGET_ARCH", archSupported, "from PlatformSettings.GetArchSupported()") _targets = " ".join(self.PlatformSettings.GetTargetsSupported()) # Generate consumable XML object- junit format JunitReport = JunitTestReport() # Keep track of failures failure_num = 0 total_num = 0 # Load plugins logging.log(edk2_logging.SECTION, "Loading plugins") pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin) if len(self.packageList) == 0: self.packageList.extend(self.PlatformSettings.GetPackages()) for pkgToRunOn in self.packageList: # # run all loaded Edk2CiBuild Plugins/Tests # logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package") logging.info(f"Running on Package: {pkgToRunOn}") ts = JunitReport.create_new_testsuite( pkgToRunOn, f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}" ) packagebuildlog_path = os.path.join(log_directory, pkgToRunOn) _, txthandle = edk2_logging.setup_txt_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) _, mdhandle = edk2_logging.setup_markdown_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) loghandle = [txthandle, mdhandle] shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() # load the package level .mu.json pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath( os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml")) if (pkg_config_file): with open(pkg_config_file, 'r') as f: pkg_config = yaml.safe_load(f) else: logging.info(f"No Pkg Config file for {pkgToRunOn}") pkg_config = dict() # check the resulting configuration config_validator.check_package_confg(pkgToRunOn, pkg_config, pluginList) # get all the defines from the package configuration if "Defines" in pkg_config: for definition_key in pkg_config["Defines"]: definition = pkg_config["Defines"][definition_key] env.SetValue(definition_key, definition, "Edk2CiBuild.py from PkgConfig yaml", False) for Descriptor in pluginList: # Get our targets targets = ["DEBUG"] if Descriptor.Obj.IsTargetDependent() and _targets: targets = self.PlatformSettings.GetTargetsSupported() for target in targets: edk2_logging.log_progress( f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --" ) total_num += 1 shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() env.SetValue("TARGET", target, "Edk2CiBuild.py before RunBuildPlugin") (testcasename, testclassname) = Descriptor.Obj.GetTestName( pkgToRunOn, env) tc = ts.create_new_testcase(testcasename, testclassname) # create the stream for the build log plugin_output_stream = edk2_logging.create_output_stream() # merge the repo level and package level for this specific plugin pkg_plugin_configuration = merge_config( self.PlatformSettings.GetPluginSettings(), pkg_config, Descriptor.descriptor) # perhaps we should ask the validator to run on the package for this target # Still need to see if the package decided this should be skipped if pkg_plugin_configuration is None or\ "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]: tc.SetSkipped() edk2_logging.log_progress( "--->Test Skipped by package! %s" % Descriptor.Name) else: try: # - package is the edk2 path to package. This means workspace/packagepath relative. # - edk2path object configured with workspace and packages path # - any additional command line args # - RepoConfig Object (dict) for the build # - PkgConfig Object (dict) # - EnvConfig Object # - Plugin Manager Instance # - Plugin Helper Obj Instance # - testcase Object used for outputing junit results # - output_stream the StringIO output stream from this plugin rc = Descriptor.Obj.RunBuildPlugin( pkgToRunOn, edk2path, pkg_plugin_configuration, env, self.plugin_manager, self.helper, tc, plugin_output_stream) except Exception as exp: exc_type, exc_value, exc_traceback = sys.exc_info() logging.critical("EXCEPTION: {0}".format(exp)) exceptionPrint = traceback.format_exception( type(exp), exp, exc_traceback) logging.critical(" ".join(exceptionPrint)) tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION") rc = 1 if (rc != 0): failure_num += 1 if (rc is None): logging.error( "--->Test Failed: %s returned NoneType" % Descriptor.Name) else: logging.error( "--->Test Failed: %s returned %d" % (Descriptor.Name, rc)) else: edk2_logging.log_progress( f"--->Test Success {Descriptor.Name} {target}") # revert to the checkpoint we created previously shell_environment.RevertBuildVars() # remove the logger edk2_logging.remove_output_stream(plugin_output_stream) # finished target loop # Finished plugin loop edk2_logging.stop_logging( loghandle) # stop the logging for this particular buildfile shell_environment.RevertBuildVars() # Finished buildable file loop JunitReport.Output( os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml")) # Print Overall Success if (failure_num != 0): logging.error("Overall Build Status: Error") edk2_logging.log_progress( f"There were {failure_num} failures out of {total_num} attempts" ) else: edk2_logging.log_progress("Overall Build Status: Success") return failure_num
def RetrieveCommandLineOptions(self, args): if args.tool_chain_tag is not None: shell_environment.GetBuildVars().SetValue("TOOL_CHAIN_TAG", args.tool_chain_tag, "Set as cli parameter") # cache this so usage within CISettings is consistant. self.ToolChainTagCacheValue = args.tool_chain_tag
def SetEnv(self): edk2_logging.log_progress("Setting up the Environment") shell_environment.GetEnvironment().set_shell_var("WORKSPACE", self.ws) shell_environment.GetBuildVars().SetValue("WORKSPACE", self.ws, "Set in SetEnv") if (self.pp is not None): shell_environment.GetEnvironment().set_shell_var( "PACKAGES_PATH", self.pp) shell_environment.GetBuildVars().SetValue("PACKAGES_PATH", self.pp, "Set in SetEnv") # process platform parameters defined in platform build file ret = self.SetPlatformEnv() if (ret != 0): logging.critical("Set Platform Env failed") return ret # set some basic defaults self.SetBasicDefaults() # Handle all the template files for workspace/conf/ Allow override TemplateDirList = [self.env.GetValue("EDK_TOOLS_PATH") ] # set to edk2 BaseTools PlatTemplatesForConf = self.env.GetValue( "CONF_TEMPLATE_DIR") # get platform defined additional path if (PlatTemplatesForConf is not None): PlatTemplatesForConf = self.mws.join(self.ws, PlatTemplatesForConf) TemplateDirList.insert(0, PlatTemplatesForConf) logging.debug( f"Platform defined override for Template Conf Files: {PlatTemplatesForConf}" ) conf_dir = os.path.join(self.ws, "Conf") conf_mgmt.ConfMgmt().populate_conf_dir(conf_dir, self.UpdateConf, TemplateDirList) # parse target file ret = self.ParseTargetFile() if (ret != 0): logging.critical("ParseTargetFile failed") return ret # parse DSC file ret = self.ParseDscFile() if (ret != 0): logging.critical("ParseDscFile failed") return ret # parse FDF file ret = self.ParseFdfFile() if (ret != 0): logging.critical("ParseFdfFile failed") return ret # set build output base envs for all builds if self.env.GetValue("OUTPUT_DIRECTORY") is None: logging.warn("OUTPUT_DIRECTORY was not found, defaulting to Build") self.env.SetValue("OUTPUT_DIRECTORY", "Build", "default from uefi_build", True) self.env.SetValue( "BUILD_OUT_TEMP", os.path.join(self.ws, self.env.GetValue("OUTPUT_DIRECTORY")), "Computed in SetEnv") target = self.env.GetValue("TARGET") self.env.SetValue( "BUILD_OUTPUT_BASE", os.path.join(self.env.GetValue("BUILD_OUT_TEMP"), target + "_" + self.env.GetValue("TOOL_CHAIN_TAG")), "Computed in SetEnv") # We have our build target now. Give platform build one more chance for target specific settings. ret = self.SetPlatformEnvAfterTarget() if (ret != 0): logging.critical("SetPlatformEnvAfterTarget failed") return ret # set the build report file self.env.SetValue( "BUILDREPORT_FILE", os.path.join(self.env.GetValue("BUILD_OUTPUT_BASE"), "BUILD_REPORT.TXT"), True) # set environment variables for the build process os.environ["EFI_SOURCE"] = self.ws return 0
def ParseCommandLineOptions(self): ''' Parses command line options. Sets up argparser specifically to get PlatformSettingsManager instance. Then sets up second argparser and passes it to child class and to PlatformSettingsManager. Finally, parses all known args and then reads the unknown args in to build vars. ''' # first argparser will only get settings manager and help will be disabled settingsParserObj = argparse.ArgumentParser(add_help=False) # instantiate the second argparser that will get passed around epilog = ''' <key>=<value> - Set an env variable for the pre/post build process BLD_*_<key>=<value> - Set a build flag for all build types. Key=value will get passed to build process BLD_<TARGET>_<key>=<value> - Set a build flag for build type of <target> Key=value will get passed to build process for given build type)''' parserObj = argparse.ArgumentParser(epilog=epilog) settingsParserObj.add_argument( '-c', '--platform_module', dest='platform_module', default="PlatformBuild.py", type=str, help= 'Provide the Platform Module relative to the current working directory.' f'This should contain a {self.GetSettingsClass().__name__} instance.' ) # get the settings manager from the provided file and load an instance settingsArg, unknown_args = settingsParserObj.parse_known_args() try: self.PlatformModule = import_module_by_file_name( os.path.abspath(settingsArg.platform_module)) self.PlatformSettings = locate_class_in_module( self.PlatformModule, self.GetSettingsClass())() except (TypeError): # Gracefully exit if the file we loaded isn't the right type class_name = self.GetSettingsClass().__name__ print( f"Unable to use {settingsArg.platform_module} as a {class_name}" ) print("Did you mean to use a different kind of invocable?") try: # If this works, we can provide help for whatever special functions # the subclass is offering. self.AddCommandLineOptions(settingsParserObj) Module = self.PlatformModule module_contents = dir(Module) # Filter through the Module, we're only looking for classes. classList = [ getattr(Module, obj) for obj in module_contents if inspect.isclass(getattr(Module, obj)) ] # Get the name of all the classes classNameList = [obj.__name__ for obj in classList] # TODO improve filter to no longer catch imports as well as declared classes imported_classes = ", ".join( classNameList) # Join the classes together print(f"The module you imported contains {imported_classes}") except Exception: # Otherwise, oh well we'll just ignore this. raise pass settingsParserObj.print_help() sys.exit(1) except (FileNotFoundError): # Gracefully exit if we can't find the file print(f"We weren't able to find {settingsArg.platform_module}") settingsParserObj.print_help() sys.exit(2) except Exception as e: print( f"Error: We had trouble loading {settingsArg.platform_module}. Is the path correct?" ) # Gracefully exit if setup doesn't go well. settingsParserObj.print_help() print(e) sys.exit(2) # now to get the big arg parser going... # first pass it to the subclass self.AddCommandLineOptions(parserObj) # next pass it to the settings manager self.PlatformSettings.AddCommandLineOptions(parserObj) default_build_config_path = os.path.join(self.GetWorkspaceRoot(), "BuildConfig.conf") # add the common stuff that everyone will need parserObj.add_argument('--build-config', dest='build_config', default=default_build_config_path, type=str, help='Provide shell variables in a file') parserObj.add_argument('--verbose', '--VERBOSE', '-v', dest="verbose", action='store_true', default=False, help='verbose') # setup sys.argv and argparse round 2 sys.argv = [sys.argv[0]] + unknown_args args, unknown_args = parserObj.parse_known_args() self.Verbose = args.verbose # give the parsed args to the subclass self.RetrieveCommandLineOptions(args) # give the parsed args to platform settings manager self.PlatformSettings.RetrieveCommandLineOptions(args) # # Look through unknown_args and BuildConfig for strings that are x=y, # set env.SetValue(x, y), # then remove this item from the list. # env = shell_environment.GetBuildVars() BuildConfig = os.path.abspath(args.build_config) for argument in unknown_args: if argument.count("=") != 1: raise RuntimeError( f"Unknown variable passed in via CLI: {argument}") tokens = argument.strip().split("=") env.SetValue(tokens[0].strip().upper(), tokens[1].strip(), "From CmdLine") unknown_args.clear() # remove the arguments we've already consumed if os.path.isfile(BuildConfig): with open(BuildConfig) as file: for line in file: stripped_line = line.strip().partition("#")[0] if len(stripped_line) == 0: continue unknown_args.append(stripped_line) for argument in unknown_args: if argument.count("=") != 1: raise RuntimeError( f"Unknown variable passed in via BuildConfig: {argument}") tokens = argument.strip().split("=") env.SetValue(tokens[0].strip().upper(), tokens[1].strip(), "From BuildConf")
def __init__(self): plugin_skip_list = ["DependencyCheck"] env = shell_environment.GetBuildVars() for plugin in plugin_skip_list: env.SetValue(plugin.upper(), "skip", "set from settings file") pass
def Go(self): log_directory = os.path.join(self.GetWorkspaceRoot(), self.GetLoggingFolderRelativeToRoot()) Edk2CiBuild.collect_python_pip_info() # # Get Package Path from config file pplist = self.PlatformSettings.GetPackagesPath( ) if self.PlatformSettings.GetPackagesPath() else [] # make Edk2Path object to handle all path operations try: edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist) except Exception as e: logging.error("Src Tree is invalid. Did you Setup correctly?") raise e logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}") logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}") logging.info(f"Package Path: {pplist}") # Bring up the common minimum environment. logging.log(edk2_logging.SECTION, "Getting Environment") (build_env, shell_env) = self_describing_environment.BootstrapEnvironment( self.GetWorkspaceRoot(), self.GetActiveScopes()) env = shell_environment.GetBuildVars() # Bind our current execution environment into the shell vars. ph = os.path.dirname(sys.executable) if " " in ph: ph = '"' + ph + '"' shell_env.set_shell_var("PYTHON_HOME", ph) # PYTHON_COMMAND is required to be set for using edk2 python builds. # todo: work with edk2 to remove the bat file and move to native python calls. # This would be better in an edk2 plugin so that it could be modified/controlled # more easily # pc = sys.executable if " " in pc: pc = '"' + pc + '"' shell_env.set_shell_var("PYTHON_COMMAND", pc) env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list), "from edk2 ci build.py") # Generate consumable XML object- junit format JunitReport = JunitTestReport() # Keep track of failures failure_num = 0 total_num = 0 # Load plugins logging.log(edk2_logging.SECTION, "Loading plugins") pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin) for pkgToRunOn in self.requested_package_list: # # run all loaded Edk2CiBuild Plugins/Tests # logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package") logging.info(f"Running on Package: {pkgToRunOn}") package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}" ts = JunitReport.create_new_testsuite(pkgToRunOn, package_class_name) packagebuildlog_path = os.path.join(log_directory, pkgToRunOn) _, txt_handle = edk2_logging.setup_txt_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) _, md_handle = edk2_logging.setup_markdown_logger( packagebuildlog_path, f"BUILDLOG_{pkgToRunOn}", logging_level=logging.DEBUG, isVerbose=True) loghandle = [txt_handle, md_handle] shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() # load the package level .ci.yaml pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath( os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml")) if (pkg_config_file): with open(pkg_config_file, 'r') as f: pkg_config = yaml.safe_load(f) else: logging.info(f"No Pkg Config file for {pkgToRunOn}") pkg_config = dict() # get all the defines from the package configuration if "Defines" in pkg_config: for definition_key in pkg_config["Defines"]: definition = pkg_config["Defines"][definition_key] env.SetValue(definition_key, definition, "Edk2CiBuild.py from PkgConfig yaml", False) # For each plugin for Descriptor in pluginList: # For each target for target in self.requested_target_list: if (target not in Descriptor.Obj.RunsOnTargetList()): continue edk2_logging.log_progress( f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --" ) total_num += 1 shell_environment.CheckpointBuildVars() env = shell_environment.GetBuildVars() env.SetValue("TARGET", target, "Edk2CiBuild.py before RunBuildPlugin") (testcasename, testclassname) = Descriptor.Obj.GetTestName( package_class_name, env) tc = ts.create_new_testcase(testcasename, testclassname) # create the stream for the build log plugin_output_stream = edk2_logging.create_output_stream() # merge the repo level and package level for this specific plugin pkg_plugin_configuration = self.merge_config( self.PlatformSettings.GetPluginSettings(), pkg_config, Descriptor.descriptor) # Still need to see if the package decided this should be skipped if pkg_plugin_configuration is None or\ "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]: tc.SetSkipped() edk2_logging.log_progress( "--->Test Skipped by package! %s" % Descriptor.Name) else: try: # - package is the edk2 path to package. This means workspace/package path relative. # - edk2path object configured with workspace and packages path # - any additional command line args # - RepoConfig Object (dict) for the build # - PkgConfig Object (dict) # - EnvConfig Object # - Plugin Manager Instance # - Plugin Helper Obj Instance # - testcase Object used for outputing junit results # - output_stream the StringIO output stream from this plugin rc = Descriptor.Obj.RunBuildPlugin( pkgToRunOn, edk2path, pkg_plugin_configuration, env, self.plugin_manager, self.helper, tc, plugin_output_stream) except Exception as exp: exc_type, exc_value, exc_traceback = sys.exc_info() logging.critical("EXCEPTION: {0}".format(exp)) exceptionPrint = traceback.format_exception( type(exp), exp, exc_traceback) logging.critical(" ".join(exceptionPrint)) tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION") rc = 1 if (rc > 0): failure_num += 1 if (rc is None): logging.error( f"--->Test Failed: {Descriptor.Name} {target} returned NoneType" ) else: logging.error( f"--->Test Failed: {Descriptor.Name} {target} returned {rc}" ) elif (rc < 0): logging.warn( f"--->Test Skipped: in plugin! {Descriptor.Name} {target}" ) else: edk2_logging.log_progress( f"--->Test Success: {Descriptor.Name} {target}" ) # revert to the checkpoint we created previously shell_environment.RevertBuildVars() # remove the logger edk2_logging.remove_output_stream(plugin_output_stream) # finished target loop # Finished plugin loop edk2_logging.stop_logging( loghandle) # stop the logging for this particular buildfile shell_environment.RevertBuildVars() # Finished buildable file loop JunitReport.Output( os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml")) # Print Overall Success if (failure_num != 0): logging.error("Overall Build Status: Error") edk2_logging.log_progress( f"There were {failure_num} failures out of {total_num} attempts" ) else: edk2_logging.log_progress("Overall Build Status: Success") return failure_num
def SetEnv(self): edk2_logging.log_progress("Setting up the Environment") shell_environment.GetEnvironment().set_shell_var("WORKSPACE", self.ws) shell_environment.GetBuildVars().SetValue("WORKSPACE", self.ws, "Set in SetEnv") if (self.pp is not None): shell_environment.GetEnvironment().set_shell_var( "PACKAGES_PATH", self.pp) shell_environment.GetBuildVars().SetValue("PACKAGES_PATH", self.pp, "Set in SetEnv") # process platform parameters defined in platform build file ret = self.SetPlatformEnv() if (ret != 0): logging.critical("Set Platform Env failed") return ret # set some basic defaults self.SetBasicDefaults() # Handle all the template files for workspace/conf/ Allow override TemplatesForConf = self.env.GetValue("CONF_TEMPLATE_DIR") if (TemplatesForConf is not None): TemplatesForConf = self.mws.join(self.ws, TemplatesForConf) logging.debug( "Platform defined override for Template Conf Files: %s", TemplatesForConf) e = conf_mgmt.ConfMgmt(self.UpdateConf, TemplatesForConf) # parse target file ret = self.ParseTargetFile() if (ret != 0): logging.critical("ParseTargetFile failed") return ret ret = e.ToolsDefConfigure() if (ret != 0): logging.critical("ParseTargetFile failed") return ret # parse DSC file ret = self.ParseDscFile() if (ret != 0): logging.critical("ParseDscFile failed") return ret # parse FDF file ret = self.ParseFdfFile() if (ret != 0): logging.critical("ParseFdfFile failed") return ret # set build output base envs for all builds self.env.SetValue( "BUILD_OUT_TEMP", os.path.join(self.ws, self.env.GetValue("OUTPUT_DIRECTORY")), "Computed in SetEnv") target = self.env.GetValue("TARGET") self.env.SetValue( "BUILD_OUTPUT_BASE", os.path.join(self.env.GetValue("BUILD_OUT_TEMP"), target + "_" + self.env.GetValue("TOOL_CHAIN_TAG")), "Computed in SetEnv") # We have our build target now. Give platform build one more chance for target specific settings. ret = self.SetPlatformEnvAfterTarget() if (ret != 0): logging.critical("SetPlatformEnvAfterTarget failed") return ret # set the build report file self.env.SetValue( "BUILDREPORT_FILE", os.path.join(self.env.GetValue("BUILD_OUTPUT_BASE"), "BUILD_REPORT.TXT"), True) # set environment variables for the build process os.environ["EFI_SOURCE"] = self.ws return 0
def ParseCommandLineOptions(self): ''' Parses command line options. Sets up argparser specifically to get PlatformSettingsManager instance. Then sets up second argparser and passes it to child class and to PlatformSettingsManager. Finally, parses all known args and then reads the unknown args in to build vars. ''' # first argparser will only get settings manager and help will be disabled settingsParserObj = argparse.ArgumentParser(add_help=False) # instantiate the second argparser that will get passed around epilog = ''' <key>=<value> - Set an env variable for the pre/post build process BLD_*_<key>=<value> - Set a build flag for all build types. Key=value will get passed to build process BLD_<TARGET>_<key>=<value> - Set a build flag for build type of <target> Key=value will get passed to build process for given build type)''' parserObj = argparse.ArgumentParser(epilog=epilog) settingsParserObj.add_argument( '-c', '--platform_module', dest='platform_module', default="PlatformBuild.py", type=str, help= 'Provide the Platform Module relative to the current working directory.' f'This should contain a {self.GetSettingsClass().__name__} instance.' ) # get the settings manager from the provided file and load an instance settingsArg, unknown_args = settingsParserObj.parse_known_args() try: self.PlatformModule = import_module_by_file_name( os.path.abspath(settingsArg.platform_module)) self.PlatformSettings = locate_class_in_module( self.PlatformModule, self.GetSettingsClass())() except (TypeError, FileNotFoundError) as e: # Gracefully exit if setup doesn't go well. try: # If this works, we can provide help for whatever special functions # the subclass is offering. self.AddCommandLineOptions(settingsParserObj) except: # If it didn't work, oh well. pass print(e) settingsParserObj.print_help() sys.exit(0) # now to get the big arg parser going... # first pass it to the subclass self.AddCommandLineOptions(parserObj) # next pass it to the settings manager self.PlatformSettings.AddCommandLineOptions(parserObj) default_build_config_path = os.path.join(self.GetWorkspaceRoot(), "BuildConfig.conf") # add the common stuff that everyone will need parserObj.add_argument('--build-config', dest='build_config', default=default_build_config_path, type=str, help='Provide shell variables in a file') parserObj.add_argument('--verbose', '--VERBOSE', '-v', dest="verbose", action='store_true', default=False, help='verbose') # setup sys.argv and argparse round 2 sys.argv = [sys.argv[0]] + unknown_args args, unknown_args = parserObj.parse_known_args() self.Verbose = args.verbose # give the parsed args to the subclass self.RetrieveCommandLineOptions(args) # give the parsed args to platform settings manager self.PlatformSettings.RetrieveCommandLineOptions(args) # # Look through unknown_args and BuildConfig for strings that are x=y, # set env.SetValue(x, y), # then remove this item from the list. # env = shell_environment.GetBuildVars() BuildConfig = os.path.abspath(args.build_config) if os.path.isfile(BuildConfig): with open(BuildConfig) as file: for line in file: stripped_line = line.strip() if not stripped_line.startswith("#"): unknown_args.append(line) i = 0 while i < len(unknown_args): unknown_arg = unknown_args[i] if (unknown_arg.count("=") == 1): tokens = unknown_arg.strip().split("=") env.SetValue(tokens[0].strip().upper(), tokens[1].strip(), "From CmdLine") del unknown_args[i] else: i += 1 if len(unknown_args) > 0: parserObj.print_help() raise RuntimeError(f"Unknown variables passed in: {unknown_args}")
def Go(self, WorkSpace, PackagesPath, PInHelper, PInManager): self.env = shell_environment.GetBuildVars() self.mws = MultipleWorkspace() self.mws.setWs(WorkSpace, PackagesPath) self.ws = WorkSpace self.pp = PackagesPath # string using os.pathsep self.Helper = PInHelper self.pm = PInManager try: edk2_logging.log_progress("Start time: {0}".format( datetime.datetime.now())) start_time = time.perf_counter() self.Helper.DebugLogRegisteredFunctions() ret = self.SetEnv() if (ret != 0): logging.critical("SetEnv failed") return ret # clean if (self.Clean): edk2_logging.log_progress("Cleaning") ret = self.CleanTree() if (ret != 0): logging.critical("Clean failed") return ret # prebuild if (self.SkipPreBuild): edk2_logging.log_progress("Skipping Pre Build") else: ret = self.PreBuild() if (ret != 0): logging.critical("Pre Build failed") return ret # Output Build Environment to File - this is mostly for debug of build # issues or adding other build features using existing variables if (self.OutputConfig is not None): edk2_logging.log_progress("Writing Build Env Info out to File") logging.debug("Found an Output Build Env File: " + self.OutputConfig) self.env.PrintAll(self.OutputConfig) if (self.env.GetValue("GATEDBUILD") is not None) and (self.env.GetValue("GATEDBUILD").upper() == "TRUE"): ShouldGatedBuildRun = self.PlatformGatedBuildShouldHappen() logging.debug("Platform Gated Build Should Run returned: %s" % str(ShouldGatedBuildRun)) if (not self.SkipBuild): self.SkipBuild = not ShouldGatedBuildRun if (not self.SkipPostBuild): self.SkipPostBuild = not ShouldGatedBuildRun # build if (self.SkipBuild): edk2_logging.log_progress("Skipping Build") else: ret = self.Build() if (ret != 0): logging.critical("Build failed") return ret # postbuild if (self.SkipPostBuild): edk2_logging.log_progress("Skipping Post Build") else: ret = self.PostBuild() if (ret != 0): logging.critical("Post Build failed") return ret # flash if (self.FlashImage): edk2_logging.log_progress("Flashing Image") ret = self.FlashRomImage() if (ret != 0): logging.critical("Flash Image failed") return ret except: logging.critical("Build Process Exception") logging.error(traceback.format_exc()) return -1 finally: end_time = time.perf_counter() elapsed_time_s = int((end_time - start_time)) edk2_logging.log_progress( "End time: {0}\t Total time Elapsed: {1}".format( datetime.datetime.now(), datetime.timedelta(seconds=elapsed_time_s))) return 0
def GetConfigurationLabel(self): env = shell_environment.GetBuildVars() target = env.GetValue("TARGET") arch = env.GetValue("TARGET_ARCH") label_parts = [target, arch] return "_".join(label_parts)