def ConfigureLogging(self):
''' Set up the logging. This function only needs to be overridden if new behavior is needed'''
logger = logging.getLogger('')
logger.setLevel(self.GetLoggingLevel("base"))
# Adjust console mode depending on mode.
edk2_logging.setup_section_level()
edk2_logging.setup_console_logging(self.GetLoggingLevel("con"))
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
txtlogfile = self.GetLoggingLevel("txt")
if (txtlogfile is not None):
logfile, filelogger = edk2_logging.setup_txt_logger(
log_directory, self.GetLoggingFileName("txt"), txtlogfile)
self.log_filename = logfile
md_log_file = self.GetLoggingLevel("md")
if (md_log_file is not None):
md_file, md_logger = edk2_logging.setup_markdown_logger(
log_directory, self.GetLoggingFileName("md"), md_log_file)
logging.info(
"Log Started: " +
datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p"))
return
def test_can_create_txt_logger(self):
test_dir = tempfile.mkdtemp()
location, txt_logger = edk2_logging.setup_txt_logger(
test_dir, "test_txt")
logging.info("Testing")
self.assertTrue(os.path.isfile(location),
"We should have created a file")
self.assertIsNot(txt_logger, None, "We created a txt logger")
edk2_logging.stop_logging(txt_logger)
def test_can_close_logger(self):
test_dir = tempfile.mkdtemp()
location, txt_logger = edk2_logging.setup_txt_logger(
test_dir, "test_close")
logging.critical("Testing")
self.assertTrue(os.path.isfile(location),
"We should have created a file")
file = open(location, "r")
num_lines = len(file.readlines())
file.close()
self.assertEqual(num_lines, 1, "We should only have one line")
edk2_logging.stop_logging(txt_logger)
logging.critical("Test 2")
file = open(location, "r")
num_lines2 = len(file.readlines())
file.close()
self.assertEqual(num_lines, num_lines2, "We should only have one line")
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
# SET PACKAGE PATH
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# Check Dependencies for Repo
for dependency in self.PlatformSettings.GetDependencies():
pplist.append(dependency["Path"])
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error(
"You need to run stewart_ci_setup to resolve all repos.")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(
f"Package Path: {self.PlatformSettings.GetPackagesPath()}")
# logging.info("mu_build version: {0}".format(pkg_resources.get_distribution("mu_build").version))
# logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
# logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Enviroment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
archSupported = " ".join(self.PlatformSettings.GetArchSupported())
env.SetValue("TARGET_ARCH", archSupported,
"from PlatformSettings.GetArchSupported()")
_targets = " ".join(self.PlatformSettings.GetTargetsSupported())
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
if len(self.packageList) == 0:
self.packageList.extend(self.PlatformSettings.GetPackages())
for pkgToRunOn in self.packageList:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
ts = JunitReport.create_new_testsuite(
pkgToRunOn,
f"Edk2CiBuild.{self.PlatformSettings.GetGroupName()}.{pkgToRunOn}"
)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txthandle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, mdhandle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txthandle, mdhandle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .mu.json
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# check the resulting configuration
config_validator.check_package_confg(pkgToRunOn, pkg_config,
pluginList)
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
for Descriptor in pluginList:
# Get our targets
targets = ["DEBUG"]
if Descriptor.Obj.IsTargetDependent() and _targets:
targets = self.PlatformSettings.GetTargetsSupported()
for target in targets:
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
pkgToRunOn, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# perhaps we should ask the validator to run on the package for this target
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc != 0):
failure_num += 1
if (rc is None):
logging.error(
"--->Test Failed: %s returned NoneType" %
Descriptor.Name)
else:
logging.error(
"--->Test Failed: %s returned %d" %
(Descriptor.Name, rc))
else:
edk2_logging.log_progress(
f"--->Test Success {Descriptor.Name} {target}")
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num
def Go(self):
log_directory = os.path.join(self.GetWorkspaceRoot(),
self.GetLoggingFolderRelativeToRoot())
Edk2CiBuild.collect_python_pip_info()
#
# Get Package Path from config file
pplist = self.PlatformSettings.GetPackagesPath(
) if self.PlatformSettings.GetPackagesPath() else []
# make Edk2Path object to handle all path operations
try:
edk2path = Edk2Path(self.GetWorkspaceRoot(), pplist)
except Exception as e:
logging.error("Src Tree is invalid. Did you Setup correctly?")
raise e
logging.info(f"Running CI Build: {self.PlatformSettings.GetName()}")
logging.info(f"WorkSpace: {self.GetWorkspaceRoot()}")
logging.info(f"Package Path: {pplist}")
# Bring up the common minimum environment.
logging.log(edk2_logging.SECTION, "Getting Environment")
(build_env,
shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
env = shell_environment.GetBuildVars()
# Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
# todo: work with edk2 to remove the bat file and move to native python calls.
# This would be better in an edk2 plugin so that it could be modified/controlled
# more easily
#
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
env.SetValue("TARGET_ARCH", " ".join(self.requested_architecture_list),
"from edk2 ci build.py")
# Generate consumable XML object- junit format
JunitReport = JunitTestReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(edk2_logging.SECTION, "Loading plugins")
pluginList = self.plugin_manager.GetPluginsOfClass(ICiBuildPlugin)
for pkgToRunOn in self.requested_package_list:
#
# run all loaded Edk2CiBuild Plugins/Tests
#
logging.log(edk2_logging.SECTION, f"Building {pkgToRunOn} Package")
logging.info(f"Running on Package: {pkgToRunOn}")
package_class_name = f"Edk2CiBuild.{self.PlatformSettings.GetName()}.{pkgToRunOn}"
ts = JunitReport.create_new_testsuite(pkgToRunOn,
package_class_name)
packagebuildlog_path = os.path.join(log_directory, pkgToRunOn)
_, txt_handle = edk2_logging.setup_txt_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
_, md_handle = edk2_logging.setup_markdown_logger(
packagebuildlog_path,
f"BUILDLOG_{pkgToRunOn}",
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txt_handle, md_handle]
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
# load the package level .ci.yaml
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".ci.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info(f"No Pkg Config file for {pkgToRunOn}")
pkg_config = dict()
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"Edk2CiBuild.py from PkgConfig yaml", False)
# For each plugin
for Descriptor in pluginList:
# For each target
for target in self.requested_target_list:
if (target not in Descriptor.Obj.RunsOnTargetList()):
continue
edk2_logging.log_progress(
f"--Running {pkgToRunOn}: {Descriptor.Name} {target} --"
)
total_num += 1
shell_environment.CheckpointBuildVars()
env = shell_environment.GetBuildVars()
env.SetValue("TARGET", target,
"Edk2CiBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(
package_class_name, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = edk2_logging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = self.merge_config(
self.PlatformSettings.GetPluginSettings(), pkg_config,
Descriptor.descriptor)
# Still need to see if the package decided this should be skipped
if pkg_plugin_configuration is None or\
"skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
edk2_logging.log_progress(
"--->Test Skipped by package! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/package path relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, pkg_plugin_configuration,
env, self.plugin_manager, self.helper, tc,
plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc > 0):
failure_num += 1
if (rc is None):
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned NoneType"
)
else:
logging.error(
f"--->Test Failed: {Descriptor.Name} {target} returned {rc}"
)
elif (rc < 0):
logging.warn(
f"--->Test Skipped: in plugin! {Descriptor.Name} {target}"
)
else:
edk2_logging.log_progress(
f"--->Test Success: {Descriptor.Name} {target}"
)
# revert to the checkpoint we created previously
shell_environment.RevertBuildVars()
# remove the logger
edk2_logging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
edk2_logging.stop_logging(
loghandle) # stop the logging for this particular buildfile
shell_environment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(self.GetWorkspaceRoot(), "Build", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
edk2_logging.log_progress(
f"There were {failure_num} failures out of {total_num} attempts"
)
else:
edk2_logging.log_progress("Overall Build Status: Success")
return failure_num