def configure_base_logging(mode="standard", sections=[]):
# Initialize logging.
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
# Adjust console mode depending on mode.
MuLogging.setup_section_level()
for section in sections:
MuLogging.get_mu_filter().addSection(section)
if mode == "vs":
MuLogging.setup_console_logging(use_color=False,
formatter="%(message)s",
logging_level=logging.DEBUG)
elif mode == 'verbose':
MuLogging.setup_console_logging(logging_level=logging.DEBUG)
elif mode == 'simple':
MuLogging.setup_console_logging(logging_level=logging.INFO)
else:
MuLogging.setup_console_logging(logging_level=logging.WARNING)
def main():
# setup main console as logger
logger = logging.getLogger('')
logger.setLevel(logging.NOTSET)
console = MuLogging.setup_console_logging(False)
logger.addHandler(console)
ErrorCode = 0
auto_fetch = False
input_config_remotes = None
# arg parse
args = get_cli_options()
if args.debug:
console.setLevel(logging.DEBUG)
logging.info("Log Started: " + datetime.datetime.strftime(
datetime.datetime.now(), "%A, %B %d, %Y %I:%M%p"))
args.cache_dir = CommonFilePathHandler(args.cache_dir)
logging.debug("OMNICACHE dir: {0}".format(args.cache_dir))
# input config file for adding new entries
if args.input_config_file is not None:
args.input_config_file = CommonFilePathHandler(args.input_config_file)
if not os.path.isfile(args.input_config_file):
logging.critical(
"Invalid -c argument given. File ({0}) isn't valid".format(
args.input_config_file))
return -4
logging.debug("Args: " + str(args))
omnicache_config = None # config object
omnicache_config_file = os.path.join(args.cache_dir, OMNICACHE_FILENAME)
if args.new:
if os.path.isdir(args.cache_dir):
logging.critical(
"--new argument given but OMNICACHE path already exists!")
return -1
InitOmnicache(args.cache_dir)
auto_fetch = True
if args.init:
if os.path.isdir(args.cache_dir):
if os.path.isfile(omnicache_config_file):
logging.debug(
"OMNICACHE already exists. No need to initialize")
else:
InitOmnicache(args.cache_dir)
auto_fetch = True
# Check to see if exists
if not os.path.isdir(args.cache_dir):
logging.critical("OMNICACHE path invalid.")
return -2
# load config
omnicache_config = OmniCacheConfig(omnicache_config_file)
os.chdir(args.cache_dir)
if (len(args.add) > 0):
auto_fetch = True
for inputdata in args.add:
if len(inputdata) == 2:
AddEntry(omnicache_config, inputdata[0], inputdata[1])
elif len(inputdata) == 3:
AddEntry(omnicache_config, inputdata[0], inputdata[1],
bool(inputdata[2]))
else:
logging.critical(
"Invalid Add Entry. Should be <name> <url> <Sync Tags optional default=False>"
)
return -3
if (args.input_config_file is not None):
(count,
input_config_remotes) = AddEntriesFromConfig(omnicache_config,
args.input_config_file)
if (count > 0):
auto_fetch = True
if len(args.remove) > 0:
for inputdata in args.remove:
RemoveEntry(omnicache_config, inputdata)
# if we need to scan
if args.scan is not None:
logging.critical("OMNICACHE is scanning the folder %s.")
if not os.path.isdir(args.scan):
logging.error("Invalid scan directory")
return -4
reposFound = dict()
# iterate through top level directories
dirs = os.listdir(args.scan)
while len(dirs) > 0:
item = dirs.pop()
itemDir = os.path.join(args.scan, item)
if os.path.isfile(itemDir):
continue
logging.info("Scanning %s for a git repo" % item)
gitDir = os.path.join(itemDir, ".git")
# Check if it's a directory or a file (submodules usually have a file instead of a folder)
if os.path.isdir(gitDir) or os.path.isfile(gitDir):
repo = MuGit.Repo(itemDir)
if repo.url:
if repo.url not in reposFound:
reposFound[repo.url] = item
else:
logging.warning(
"Skipping previously found repo at %s with url %s"
% (item, repo.url))
else: # if repo.url is none
logging.error("Url not found for git repo at: %s" %
itemDir)
# check for submodules
if repo.submodules:
for submodule in repo.submodules:
dirs.append(os.path.join(item, submodule))
else:
logging.error("Git repo not found at %s" % itemDir)
# go through all the URL's I found
for url in reposFound:
omnicache_config.Add(reposFound[url], url)
omnicache_config.Save()
if (args.fetch or (auto_fetch and not args.no_fetch)):
logging.critical("Updating OMNICACHE")
# as an optimization, if input config file provided, only fetch remotes specified in input config
# otherwise, fetch all remotes in the OmniCache
if (input_config_remotes is not None):
remotes = (x["name"] for x in input_config_remotes)
else:
remotes = omnicache_config.remotes.keys()
for remote in remotes:
ret = FetchEntry(omnicache_config.remotes[remote]["name"],
("tag" in omnicache_config.remotes[remote]))
if (ret != 0) and (ErrorCode == 0):
ErrorCode = ret
if args.list:
ret = ConsistencyCheckCacheConfig(omnicache_config)
if (ret != 0) and (ErrorCode == 0):
ErrorCode = ret
print("List OMNICACHE content\n")
if len(omnicache_config.remotes) == 0:
logging.warning("No Remotes to show")
for remote in omnicache_config.remotes.values():
rstring = "Name: {0}\n Url: {1}\n Sync Tags: {2}".format(
remote["name"], remote["url"], ("tag" in remote))
print(" " + rstring + "\n\n")
print("To use your OMNICACHE with Project Mu builds set the env variable:")
print("set OMNICACHE_PATH=" + args.cache_dir)
return ErrorCode
if __name__ == "__main__":
# get the root directory of mu_plus
scriptDir = os.path.dirname(os.path.realpath(__file__))
driverDir = os.path.join(scriptDir, "Driver")
rootDir = os.path.dirname(scriptDir)
os.chdir(rootDir) # set ourselves to there
rootPath = os.path.realpath(os.path.join(rootDir, "ci.mu.yaml")) # get our yaml file
args = sys.argv
if len(args) != 2:
raise RuntimeError("You need to include your API key as the first argument")
api_key = args[1]
MuLogging.setup_console_logging(use_color=False, logging_level=logging.DEBUG)
# move the EFI's we generated to a folder to upload
NugetPath = os.path.join(rootDir, "MU_BASECORE", "BaseTools", "NugetPublishing")
NugetFilePath = os.path.join(NugetPath, "NugetPublishing.py")
if not os.path.exists(NugetFilePath):
raise FileNotFoundError(NugetFilePath)
logging.info("Running NugetPackager")
output_dir = os.path.join(rootDir, "Build", ".NugetOutput")
try:
if os.path.exists(output_dir):
shutil.rmtree(output_dir, ignore_errors=True)
os.makedirs(output_dir)
except:
logging.error("Ran into trouble getting Nuget Output Path setup")
def main():
# Parse command line arguments
PROJECT_SCOPES = ("project_mu", )
buildArgs = get_mu_config()
mu_config_filepath = os.path.abspath(buildArgs.mu_config)
if mu_config_filepath is None or not os.path.isfile(mu_config_filepath):
raise FileNotFoundError("Invalid path to mu.json file for build: ",
mu_config_filepath)
# have a build config file
with open(mu_config_filepath, 'r') as mu_config_file:
mu_config = yaml.safe_load(mu_config_file)
WORKSPACE_PATH = os.path.realpath(
os.path.join(os.path.dirname(mu_config_filepath),
mu_config["RelativeWorkspaceRoot"]))
# Setup the logging to the file as well as the console
MuLogging.clean_build_logs(WORKSPACE_PATH)
buildlog_path = os.path.join(WORKSPACE_PATH, "Build", "BuildLogs")
logging.getLogger("").setLevel(logging.NOTSET)
filename = "BUILDLOG_MASTER"
MuLogging.setup_section_level()
MuLogging.setup_txt_logger(buildlog_path, filename)
MuLogging.setup_markdown_logger(buildlog_path, filename)
MuLogging.setup_console_logging(use_azure_colors=buildArgs.use_azure_color,
use_color=buildArgs.color_enabled,
logging_level=logging.WARNING)
# Get scopes from config file
if "Scopes" in mu_config:
PROJECT_SCOPES += tuple(mu_config["Scopes"])
omnicache_path = None
if "ReferencePath" in mu_config:
omnicache_path = mu_config["ReferencePath"]
if buildArgs.omnicache_path is not None:
omnicache_path = buildArgs.omnicache_path
# SET PACKAGE PATH
#
# Get Package Path from config file
pplist = list()
if (mu_config["RelativeWorkspaceRoot"] != ""):
# this package is not at workspace root.
# Add self
pplist.append(os.path.dirname(mu_config_filepath))
# Include packages from the config file
if "PackagesPath" in mu_config:
for a in mu_config["PackagesPath"]:
pplist.append(a)
# Check Dependencies for Repo
if "Dependencies" in mu_config:
logging.log(MuLogging.SECTION, "Resolving Git Repos")
pplist.extend(
RepoResolver.resolve_all(WORKSPACE_PATH,
mu_config["Dependencies"],
ignore=buildArgs.git_ignore,
force=buildArgs.git_force,
update_ok=buildArgs.git_update,
omnicache_dir=omnicache_path))
# make Edk2Path object to handle all path operations
edk2path = Edk2Path(WORKSPACE_PATH, pplist)
logging.info("Running ProjectMu Build: {0}".format(mu_config["Name"]))
logging.info("WorkSpace: {0}".format(edk2path.WorkspacePath))
logging.info("Package Path: {0}".format(edk2path.PackagePathList))
logging.info("mu_build version: {0}".format(
pkg_resources.get_distribution("mu_build").version))
logging.info("mu_python_library version: " +
pkg_resources.get_distribution("mu_python_library").version)
logging.info("mu_environment version: " +
pkg_resources.get_distribution("mu_environment").version)
# which package to build
packageList = mu_config["Packages"]
#
# If mu pk list supplied lets see if they are a file system path
# If so convert to edk2 relative path
#
#
if (len(buildArgs.pkglist) > 0):
packageList = [] # clear it
for mu_pk_path in buildArgs.pkglist:
# if abs path lets convert
if os.path.isabs(mu_pk_path):
temp = edk2path.GetEdk2RelativePathFromAbsolutePath(mu_pk_path)
if (temp is not None):
packageList.append(temp)
else:
logging.critical(
"pkg-dir invalid absolute path: {0}".format(mu_pk_path))
raise FileNotFoundError("Invalid Package Path")
else:
# Check if relative path
temp = os.path.join(os.getcwd(), mu_pk_path)
temp = edk2path.GetEdk2RelativePathFromAbsolutePath(temp)
if (temp is not None):
packageList.append(temp)
else:
logging.critical(
"pkg-dir invalid relative path: {0}".format(mu_pk_path))
raise FileNotFoundError("Invalid Package Path")
# Bring up the common minimum environment.
logging.log(MuLogging.SECTION, "Bootstrapping Enviroment")
(build_env, shell_env) = SelfDescribingEnvironment.BootstrapEnvironment(
edk2path.WorkspacePath, PROJECT_SCOPES)
CommonBuildEntry.update_process(edk2path.WorkspacePath, PROJECT_SCOPES)
env = ShellEnvironment.GetBuildVars()
archSupported = " ".join(mu_config["ArchSupported"])
env.SetValue("TARGET_ARCH", archSupported, "Platform Hardcoded")
# Generate consumable XML object- junit format
JunitReport = MuJunitReport()
# Keep track of failures
failure_num = 0
total_num = 0
# Load plugins
logging.log(MuLogging.SECTION, "Loading plugins")
pluginManager = PluginManager.PluginManager()
failedPlugins = pluginManager.SetListOfEnvironmentDescriptors(
build_env.plugins)
if failedPlugins:
logging.critical("One or more plugins failed to load. Halting build.")
for a in failedPlugins:
logging.error("Failed Plugin: {0}".format(a["name"]))
raise RuntimeError("One or more plugins failed to load.")
helper = PluginManager.HelperFunctions()
if (helper.LoadFromPluginManager(pluginManager) > 0):
raise RuntimeError("One or more helper plugins failed to load.")
pluginList = pluginManager.GetPluginsOfClass(PluginManager.IMuBuildPlugin)
# Check to make sure our configuration is valid
ConfigValidator.check_mu_confg(mu_config, edk2path, pluginList)
for pkgToRunOn in packageList:
#
# run all loaded MuBuild Plugins/Tests
#
logging.log(MuLogging.SECTION,
"Building {0} Package".format(pkgToRunOn))
logging.info("Running on Package: {0}".format(pkgToRunOn))
ts = JunitReport.create_new_testsuite(
pkgToRunOn, "MuBuild.{0}.{1}".format(mu_config["GroupName"],
pkgToRunOn))
packagebuildlog_path = os.path.join(buildlog_path, pkgToRunOn)
_, txthandle = MuLogging.setup_txt_logger(
packagebuildlog_path,
"BUILDLOG_{0}".format(pkgToRunOn),
logging_level=logging.DEBUG,
isVerbose=True)
_, mdhandle = MuLogging.setup_markdown_logger(
packagebuildlog_path,
"BUILDLOG_{0}".format(pkgToRunOn),
logging_level=logging.DEBUG,
isVerbose=True)
loghandle = [txthandle, mdhandle]
ShellEnvironment.CheckpointBuildVars()
env = ShellEnvironment.GetBuildVars()
# load the package level .mu.json
pkg_config_file = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
os.path.join(pkgToRunOn, pkgToRunOn + ".mu.yaml"))
if (pkg_config_file):
with open(pkg_config_file, 'r') as f:
pkg_config = yaml.safe_load(f)
else:
logging.info("No Pkg Config file for {0}".format(pkgToRunOn))
pkg_config = dict()
# check the resulting configuration
ConfigValidator.check_package_confg(pkgToRunOn, pkg_config, pluginList)
# get all the defines from the package configuration
if "Defines" in pkg_config:
for definition_key in pkg_config["Defines"]:
definition = pkg_config["Defines"][definition_key]
env.SetValue(definition_key, definition,
"MuBuild.py from PkgConfig yaml", False)
for Descriptor in pluginList:
# Get our targets
targets = ["DEBUG"]
if Descriptor.Obj.IsTargetDependent() and "Targets" in mu_config:
targets = mu_config["Targets"]
for target in targets:
MuLogging.log_progress("--Running {2}: {0} {1} --".format(
Descriptor.Name, target, pkgToRunOn))
total_num += 1
ShellEnvironment.CheckpointBuildVars()
env = ShellEnvironment.GetBuildVars()
env.SetValue("TARGET", target,
"MuBuild.py before RunBuildPlugin")
(testcasename,
testclassname) = Descriptor.Obj.GetTestName(pkgToRunOn, env)
tc = ts.create_new_testcase(testcasename, testclassname)
# create the stream for the build log
plugin_output_stream = MuLogging.create_output_stream()
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(
mu_config, pkg_config, Descriptor.descriptor)
# perhaps we should ask the validator to run on the
# Check if need to skip this particular plugin
if "skip" in pkg_plugin_configuration and pkg_plugin_configuration[
"skip"]:
tc.SetSkipped()
MuLogging.log_progress("--->Test Skipped! %s" %
Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
# - output_stream the StringIO output stream from this plugin
rc = Descriptor.Obj.RunBuildPlugin(
pkgToRunOn, edk2path, sys.argv, mu_config,
pkg_plugin_configuration, env, pluginManager,
helper, tc, plugin_output_stream)
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(
type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp),
"UNEXPECTED EXCEPTION")
rc = 1
if (rc != 0):
failure_num += 1
if (rc is None):
logging.error(
"--->Test Failed: %s returned NoneType" %
Descriptor.Name)
else:
logging.error("--->Test Failed: %s returned %d" %
(Descriptor.Name, rc))
else:
MuLogging.log_progress(
"--->Test Success {0} {1}".format(
Descriptor.Name, target))
# revert to the checkpoint we created previously
ShellEnvironment.RevertBuildVars()
# remove the logger
MuLogging.remove_output_stream(plugin_output_stream)
# finished target loop
# Finished plugin loop
MuLogging.stop_logging(
loghandle) # stop the logging for this particular buildfile
ShellEnvironment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(
os.path.join(WORKSPACE_PATH, "Build", "BuildLogs", "TestSuites.xml"))
# Print Overall Success
if (failure_num != 0):
logging.error("Overall Build Status: Error")
MuLogging.log_progress(
"There were {0} failures out of {1} attempts".format(
failure_num, total_num))
else:
MuLogging.log_progress("Overall Build Status: Success")
sys.exit(failure_num)
def test_can_create_console_logger(self):
console_logger = MuLogging.setup_console_logging(False, False)
self.assertIsNot(console_logger, None, "We created a console logger")
MuLogging.stop_logging(console_logger)