def Push(self, nuPackage, apikey):
if(not os.path.isfile(nuPackage)):
raise Exception("Invalid file path for NuPkg file")
logging.debug("Pushing %s file to server %s" % (nuPackage, self.ConfigData["server_url"]))
cmd = NugetDependency.GetNugetCmd()
cmd += ["push", nuPackage]
cmd += ["-Verbosity", "detailed"]
# cmd += ["-NonInteractive"]
cmd += ["-Source", self.ConfigData["server_url"]]
cmd += ["-ApiKey", apikey]
output_buffer = StringIO()
ret = RunCmd(cmd[0], " ".join(cmd[1:]), outstream=output_buffer)
if(ret != 0):
# Rewind the buffer and capture the contents.
output_buffer.seek(0)
output_contents = output_buffer.read()
# Check for the API message.
if "API key is invalid".lower() in output_contents.lower():
logging.critical("API key is invalid. Please use --ApiKey to provide a valid key.")
# Generic error.
logging.error("Failed on nuget commend. RC = 0x%x" % ret)
return ret
def PackageMsFmpHeader(InputBin,
OutputBin,
VersionInt,
LsvInt,
DepList=[]):
logging.debug("CapsulePackage: Fmp Header")
params = "-o " + OutputBin
params = params + " --version " + hex(VersionInt).rstrip("L")
params = params + " --lsv " + hex(LsvInt)
params = params + " -p " + InputBin + " -v"
#append depedency if supplied
for dep in DepList:
depGuid = dep[0]
depIndex = int(dep[1])
depMinVer = hex(dep[2])
depFlag = hex(dep[3])
logging.debug(
"Adding a Dependency:\n\tFMP Guid: %s \nt\tFmp Descriptor Index: %d \n\tFmp DepVersion: %s \n\tFmp Flags: %s\n"
% (depGuid, depIndex, depMinVer, depFlag))
params += " --dep " + depGuid + " " + str(
depIndex) + " " + depMinVer + " " + depFlag
ret = RunCmd("genmspayloadheader.exe", params)
if (ret != 0):
raise Exception("GenMsPayloadHeader Failed with errorcode %d" %
ret)
return ret
def MakeCat(self, OutputCatFile, PathToInf2CatTool=None):
# Find Inf2Cat tool
if (PathToInf2CatTool is None):
PathToInf2CatTool = FindToolInWinSdk("Inf2Cat.exe")
# check if exists
if not os.path.exists(PathToInf2CatTool):
raise Exception(
"Can't find Inf2Cat on this machine. Please install the Windows 10 WDK - "
"https://developer.microsoft.com/en-us/windows/hardware/windows-driver-kit"
)
# Adjust for spaces in the path (when calling the command).
if " " in PathToInf2CatTool:
PathToInf2CatTool = '"' + PathToInf2CatTool + '"'
OutputFolder = os.path.dirname(OutputCatFile)
# Make Cat file
cmd = "/driver:. /os:" + self.OperatingSystem + "_" + self.Arch + " /verbose"
ret = RunCmd(PathToInf2CatTool, cmd, workingdir=OutputFolder)
if (ret != 0):
raise Exception("Creating Cat file Failed with errorcode %d" % ret)
if (not os.path.isfile(OutputCatFile)):
raise Exception("CAT file (%s) not created" % OutputCatFile)
return 0
def GetCommitHashes(root_dir: os.PathLike):
# Recursively looks at every .git and gets the commit from there
search_path = os.path.join(root_dir, "**", ".git")
search = glob.iglob(search_path, recursive=True)
found_repos = {}
cmd_args = "rev-parse HEAD"
for git_path in search:
git_path_dir = os.path.dirname(git_path)
if git_path_dir == root_dir:
git_repo_name = "MU_PLUS"
else:
_, git_repo_name = os.path.split(git_path_dir)
git_repo_name = git_repo_name.upper()
if git_repo_name in found_repos:
raise RuntimeError("we've already found this repo before " +
git_repo_name)
# read the git hash for this repo
return_buffer = StringIO()
RunCmd("git",
cmd_args,
workingdir=git_path_dir,
outstream=return_buffer)
commit_hash = return_buffer.getvalue().strip()
return_buffer.close()
found_repos[git_repo_name] = commit_hash
return found_repos
def Pack(self, version, OutputDirectory, ContentDir, RelNotesText=None):
self.NewVersion = version
# content must be absolute path in nuspec otherwise it is assumed
# relative to nuspec file.
cdir = os.path.abspath(ContentDir)
# make nuspec file
xmlstring = self._MakeNuspecXml(cdir, RelNotesText)
nuspec = os.path.join(OutputDirectory, self.Name + ".nuspec")
self.TempFileToDelete.append(nuspec)
f = open(nuspec, "wb")
f.write(xmlstring)
f.close()
# run nuget
cmd = NugetDependency.GetNugetCmd()
cmd += ["pack", nuspec]
cmd += ["-OutputDirectory", '"' + OutputDirectory + '"']
cmd += ["-Verbosity", "detailed"]
# cmd += ["-NonInteractive"]
ret = RunCmd(cmd[0], " ".join(cmd[1:]))
if(ret != 0):
logging.error("Failed on nuget commend. RC = 0x%x" % ret)
return ret
self.NuPackageFile = os.path.join(OutputDirectory, self._GetNuPkgFileName(self.NewVersion))
self.TempFileToDelete.append(self.NuPackageFile)
return ret
def MakeCat(self, OutputCatFile, PathToInf2CatTool=None):
# Find Inf2Cat tool
if (PathToInf2CatTool is None):
PathToInf2CatTool = os.path.join(os.getenv("ProgramFiles(x86)"),
"Windows Kits", "10", "bin",
"x86", "Inf2Cat.exe")
if not os.path.exists(PathToInf2CatTool):
logging.debug("Windows Kit 10 not Found....trying 8.1")
# Try 8.1 kit
PathToInf2CatTool.replace("10", "8.1")
# check if exists
if not os.path.exists(PathToInf2CatTool):
raise Exception(
"Can't find Inf2Cat on this machine. Please install the Windows 10 WDK - "
"https://developer.microsoft.com/en-us/windows/hardware/windows-driver-kit"
)
# Adjust for spaces in the path (when calling the command).
if " " in PathToInf2CatTool:
PathToInf2CatTool = '"' + PathToInf2CatTool + '"'
OutputFolder = os.path.dirname(OutputCatFile)
# Make Cat file
cmd = "/driver:. /os:" + self.OperatingSystem + "_" + self.Arch + " /verbose"
ret = RunCmd(PathToInf2CatTool, cmd, workingdir=OutputFolder)
if (ret != 0):
raise Exception("Creating Cat file Failed with errorcode %d" % ret)
if (not os.path.isfile(OutputCatFile)):
raise Exception("CAT file (%s) not created" % OutputCatFile)
return 0
def PlatformPostBuild(self):
rc = 0
os.environ["CMOCKA_MESSAGE_OUTPUT"] = self.env.GetValue("TEST_OUTPUT_FORMAT", "xml")
logging.log(MuLogging.get_section_level(), "Run Host based Unit Tests")
path = self.env.GetValue("BUILD_OUTPUT_BASE")
for arch in self.env.GetValue("TARGET_ARCH").split():
logging.log( MuLogging.get_subsection_level(), "Testing for architecture: " + arch)
cp = os.path.join(path, arch)
for old_result in glob.iglob(os.path.join(cp, "*.result.xml")):
os.remove(old_result)
testList = glob.glob(os.path.join(cp, "*Test*.exe"))
for test in testList:
os.environ["CMOCKA_XML_FILE"] = test + ".%g." + arch + ".result.xml"
ret = RunCmd('"' + test + '"', "", workingdir=cp)
if(ret != 0):
logging.error("UnitTest Execution Error: " + os.path.basename(test))
rc = ret
else:
logging.info("UnitTest Completed: " + os.path.basename(test))
file_match_pattern = test + ".*." + arch + ".result.xml"
xml_results_list = glob.glob(file_match_pattern)
for xml_result_file in xml_results_list:
root = xml.etree.ElementTree.parse(xml_result_file).getroot()
for suite in root:
for case in suite:
for result in case:
if result.tag == 'failure':
logging.warning("%s Test Failed" % os.path.basename(test))
logging.warning(" %s - %s" % (case.attrib['name'], result.text))
return rc
def fetch(self):
package_name = self.name
#
# Before trying anything with Nuget feeds,
# check to see whether the package is already in
# our local cache. If it is, we avoid a lot of
# time and network cost by copying it directly.
#
if self._fetch_from_cache(package_name):
# We successfully found the package in the cache.
# The published path may change now that the package has been unpacked.
# Bail.
self.published_path = self.compute_published_path()
return
#
# If we are still here, the package wasn't in the cache.
# We need to ask Nuget to find it.
#
#
# First, fetch the contents of the package.
#
temp_directory = self.get_temp_dir()
cmd = NugetDependency.GetNugetCmd()
cmd += ["install", package_name]
cmd += ["-Source", self.source]
cmd += ["-ExcludeVersion"]
cmd += ["-Version", self.version]
cmd += ["-Verbosity", "detailed"]
cmd += ["-OutputDirectory", '"' + temp_directory + '"']
RunCmd(cmd[0], " ".join(cmd[1:]))
#
# Next, copy the contents of the package to the
# final resting place.
#
# Depending on packaging, the package content will be in one of two
# possible locations:
# 1. temp_directory\package_name\package_name\
# 2. temp_directory\package_name\
#
source_dir = os.path.join(temp_directory, package_name, package_name)
if not os.path.isdir(source_dir):
source_dir = os.path.join(temp_directory, package_name)
shutil.move(source_dir, self.contents_dir)
#
# Add a file to track the state of the dependency.
#
self.update_state_file()
#
# Finally, delete the temp directory.
#
self._clean_directory(temp_directory)
# The published path may change now that the package has been unpacked.
self.published_path = self.compute_published_path()
def PackageFmpCapsuleHeader(InputBin, OutputBin, FmpGuid):
logging.debug("CapsulePackage: Fmp Capsule Header")
params = "-o " + OutputBin
params = params + " -p " + InputBin + " " + FmpGuid + " 1 0 -V"
ret = RunCmd("genfmpcap.exe", params)
if (ret != 0):
raise Exception("GenFmpCap Failed with errorcode" % ret)
return ret
def _get_branch(self):
return_buffer = StringIO()
params = "rev-parse --abbrev-ref HEAD"
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
return p1
def _get_url(self, remote="origin"):
return_buffer = StringIO()
params = "config --get remote.{0}.url".format(remote)
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
return p1
def clone_from(self,
url,
to_path,
progress=None,
env=None,
shallow=False,
reference=None,
**kwargs):
_logger = logging.getLogger("git.repo")
_logger.debug("Cloning {0} into {1}".format(url, to_path))
# make sure we get the commit if
# use run command from utilities
cmd = "git"
params = ["clone"]
if shallow:
params.append("--shallow-submodules")
if reference:
params.append("--reference %s" % reference)
else:
params.append(
"--recurse-submodules"
) # if we don't have a reference we can just recurse the submodules
params.append(url)
params.append(to_path)
# Combine all the parameters together
param_string = " ".join(params)
ret = RunCmd(cmd, param_string)
if ret != 0:
logging.error("ERROR CLONING ")
return None
# if we have a reference path we must init the submodules
if reference:
params = ["submodule", "update", "--init", "--recursive"]
params.append("--reference %s" % reference)
param_string = " ".join(params)
ret = RunCmd(cmd, param_string)
return Repo(to_path)
def _get_bare(self):
return_buffer = StringIO()
params = "rev-parse --is-bare-repository"
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
if p1.lower() == "true":
return True
else:
return False
def _get_head(self):
return_buffer = StringIO()
params = "rev-parse HEAD"
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
head = ObjectDict()
head.set("commit", p1)
return head
def _get_submodule_list(self):
submodule_list = []
return_buffer = StringIO()
params = "config --file .gitmodules --get-regexp path"
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
if (len(p1) > 0):
submodule_list = p1.split("\n")
for i in range(0, len(submodule_list)):
submodule_list[i] = submodule_list[i].split(' ')[1]
return submodule_list
def GetReleaseNote():
cmd = "log --format=%B -n 1 HEAD"
return_buffer = StringIO()
if (RunCmd("git", cmd, outstream=return_buffer) == 0):
# Seek to the beginning of the output buffer and capture the output.
return_buffer.seek(0)
return_string = return_buffer.read(155).replace(
"\n", " ") # read the first 155 characters and replace the
return_buffer.close()
# TODO: figure out if there was more input and append a ... if needed
return return_string.strip()
else:
raise RuntimeError("Unable to read release notes")
def _get_remotes(self):
return_buffer = StringIO()
params = "remote"
new_remotes = ObjectDict()
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
remote_list = p1.split("\n")
for remote in remote_list:
url = ObjectDict()
url.set("url", self._get_url(remote))
setattr(new_remotes, remote, url)
return new_remotes
def PackageCapsuleHeader(InputBin, OutputBin, FmpDeviceGuid=None):
logging.debug("CapsulePackage: Final Capsule Header")
if(FmpDeviceGuid == None):
logging.debug("CapsulePackage: Using default industry standard FMP guid")
FmpDeviceGuid = "6dcbd5ed-e82d-4c44-bda1-7194199ad92a"
params = "-o " + OutputBin
params = params + " -g " + FmpDeviceGuid
params = params + " --capsule -v -f " + InputBin
params = params + " --capFlag PersistAcrossReset --capFlag InitiateReset"
ret = RunCmd("genfv", params)
if(ret != 0):
raise Exception("GenFv Failed with errorcode" % ret)
return ret
def GetLatestNugetVersion(package_name, source=None):
cmd = NugetDependency.GetNugetCmd()
cmd += ["list"]
cmd += [package_name]
if source is not None:
cmd += ["-Source", source]
return_buffer = StringIO()
if (RunCmd(cmd[0], " ".join(cmd[1:]), outstream=return_buffer) == 0):
# Seek to the beginning of the output buffer and capture the output.
return_buffer.seek(0)
return_string = return_buffer.read()
return_buffer.close()
return return_string.strip().strip(package_name).strip()
else:
return "0.0.0.0"
def _get_dirty(self):
return_buffer = StringIO()
params = "status --short"
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
if len(p1) > 0:
return True
return_buffer = StringIO()
params = "log --branches --not --remotes --decorate --oneline"
RunCmd("git", params, workingdir=self._path, outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
return_buffer.close()
if len(p1) > 0:
return True
return False
def pull(self):
return_buffer = StringIO()
params = "pull"
ret = RunCmd("git",
params,
workingdir=self._path,
outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
if ret != 0:
self._logger.error(p1)
return False
return True
def DownloadNugetPackageVersion(package_name: str,
version: str,
destination: os.PathLike,
source=None):
cmd = NugetDependency.GetNugetCmd()
cmd += ["install", package_name]
if source is not None:
cmd += ["-Source", source]
cmd += ["-ExcludeVersion"]
cmd += ["-Version", version]
cmd += ["-Verbosity", "detailed"]
cmd += ["-OutputDirectory", '"' + destination + '"']
ret = RunCmd(cmd[0], " ".join(cmd[1:]))
if ret != 0:
return False
else:
return True
def checkout(self, branch=None, commit=None):
return_buffer = StringIO()
if branch is not None:
params = "checkout %s" % branch
elif commit is not None:
params = "checkout %s" % commit
ret = RunCmd("git",
params,
workingdir=self._path,
outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
if ret != 0:
self._logger.debug(p1)
return False
return True
def submodule(self, command, *args):
self._logger.debug("Calling command on submodule {0} with {1}".format(
command, args))
return_buffer = StringIO()
flags = " ".join(args)
params = "submodule {0} {1}".format(command, flags)
ret = RunCmd("git",
params,
workingdir=self._path,
outstream=return_buffer)
p1 = return_buffer.getvalue().strip()
if ret != 0:
self._logger.error(p1)
return False
return True
def FindWithVsWhere(products: str = "*"):
cmd = "-latest -nologo -all -property installationPath"
vs_where_path = GetVsWherePath()
if vs_where_path is None:
logging.warning("We weren't able to find VSWhere")
return (1, None)
if (products is not None):
cmd += " -products " + products
a = StringIO()
ret = RunCmd(vs_where_path, cmd, outstream=a)
if (ret != 0):
a.close()
return (ret, None)
p1 = a.getvalue().strip()
a.close()
if (len(p1.strip()) > 0):
return (0, p1)
return (ret, None)
def _fetch_from_cache(self, package_name):
result = False
#
# We still need to use Nuget to figure out where the
# "global-packages" cache is on this machine.
#
if NugetDependency.global_cache_path is None:
cmd = NugetDependency.GetNugetCmd()
cmd += ["locals", "global-packages", "-list"]
return_buffer = StringIO()
if (RunCmd(cmd[0], " ".join(cmd[1:]),
outstream=return_buffer) == 0):
# Seek to the beginning of the output buffer and capture the output.
return_buffer.seek(0)
return_string = return_buffer.read()
NugetDependency.global_cache_path = return_string.strip(
).strip("global-packages: ")
#
# If the path couldn't be found, we can't do anything else.
#
if not os.path.isdir(NugetDependency.global_cache_path):
logging.info(
"Could not determine Nuget global packages cache location.")
return False
#
# Now, try to locate our actual cache path
nuget_version = NugetDependency.normalize_version(self.version)
cache_search_path = os.path.join(NugetDependency.global_cache_path,
package_name.lower(), nuget_version,
package_name)
if os.path.isdir(cache_search_path):
logging.info(
"Local Cache found for Nuget package '%s'. Skipping fetch.",
package_name)
shutil.copytree(cache_search_path, self.contents_dir)
self.update_state_file()
result = True
return result
def GetReleaseForCommit(commit_hash: str):
git_dir = os.path.dirname(SCRIPT_PATH)
cmd_args = ["log", '--format="%h %D"', "--all", "-n 100"]
return_buffer = StringIO()
RunCmd("git",
" ".join(cmd_args),
workingdir=git_dir,
outstream=return_buffer)
return_buffer.seek(0)
results = return_buffer.readlines()
return_buffer.close()
log_re = re.compile(r'release/(\d{6})')
for log_item in results:
commit = log_item[:11]
branch = log_item[11:].strip()
if len(branch) == 0:
continue
match = log_re.search(branch)
if match:
return match.group(1)
raise RuntimeError(
"We couldn't find the release branch that we correspond to")
def minimum_env_init(my_workspace_path, my_project_scope):
# TODO: Check the Git version against minimums.
# Check the Python version against minimums.
cur_py = "%d.%d.%d" % sys.version_info[:3]
VersionAggregator.GetVersionAggregator().ReportVersion(
"Python", cur_py, VersionAggregator.VersionTypes.TOOL)
soft_min_py = "3.7"
hard_min_py = "3.6"
if version_compare(hard_min_py, cur_py) > 0:
raise RuntimeError(
"Please upgrade Python! Current version is %s. Minimum is %s." %
(cur_py, hard_min_py))
if version_compare(soft_min_py, cur_py) > 0:
logging.error(
"Please upgrade Python! Current version is %s. Recommended minimum is %s."
% (cur_py, soft_min_py))
return_buffer = StringIO()
RunCmd("git", "--version", outstream=return_buffer)
git_version = return_buffer.getvalue().strip()
return_buffer.close()
VersionAggregator.GetVersionAggregator().ReportVersion(
"Git", git_version, VersionAggregator.VersionTypes.TOOL)
min_git = "2.11.0"
# This code is highly specific to the return value of "git version"...
cur_git = ".".join(git_version.split(' ')[2].split(".")[:3])
if version_compare(min_git, cur_git) > 0:
raise RuntimeError(
"Please upgrade Git! Current version is %s. Minimum is %s." %
(cur_git, min_git))
# Initialized the build environment.
return SelfDescribingEnvironment.BootstrapEnvironment(
my_workspace_path, my_project_scope)
def setup_process(my_workspace_path,
my_project_scope,
my_required_repos,
force_it=False,
cache_path=None):
def log_lines(level, lines):
for line in lines.split("\n"):
if line != "":
logging.log(level, line)
# Pre-setup cleaning if "--force" is specified.
if force_it:
try:
# Clean and reset the main repo.
MuLogging.log_progress("## Cleaning the root repo...")
cmd_with_output('git reset --hard', my_workspace_path)
log_lines(logging.INFO,
cmd_with_output('git clean -xffd', my_workspace_path))
MuLogging.log_progress("Done.\n")
# Clean any submodule repos.
if my_required_repos:
for required_repo in my_required_repos:
MuLogging.log_progress(
"## Cleaning Git repository: %s..." % required_repo)
required_repo_path = os.path.normpath(
os.path.join(my_workspace_path, required_repo))
cmd_with_output('git reset --hard', required_repo_path)
log_lines(
logging.INFO,
cmd_with_output('git clean -xffd', required_repo_path))
MuLogging.log_progress("Done.\n")
except RuntimeError as e:
logging.error("FAILED!\n")
logging.error("Error while trying to clean the environment!")
log_lines(logging.ERROR, str(e))
return
# Grab the remaining Git repos.
if my_required_repos:
# Git Repos: STEP 1 --------------------------------------
# Make sure that the repos are all synced.
try:
MuLogging.log_progress("## Syncing Git repositories: %s..." %
", ".join(my_required_repos))
cmd_with_output(
'git submodule sync -- ' + " ".join(my_required_repos),
my_workspace_path)
MuLogging.log_progress("Done.\n")
except RuntimeError as e:
logging.error("FAILED!\n")
logging.error("Error while trying to synchronize the environment!")
log_lines(logging.ERROR, str(e))
return
# Git Repos: STEP 2 --------------------------------------
# Iterate through all repos and see whether they should be fetched.
for required_repo in my_required_repos:
try:
MuLogging.log_progress("## Checking Git repository: %s..." %
required_repo)
# Git Repos: STEP 2a ---------------------------------
# Need to determine whether to skip this repo.
required_repo_path = os.path.normpath(
os.path.join(my_workspace_path, required_repo))
skip_repo = False
# If the repo exists (and we're not forcing things) make
# sure that it's not in a "dirty" state.
if os.path.exists(required_repo_path) and not force_it:
git_data = cmd_with_output('git diff ' + required_repo,
my_workspace_path)
# If anything was returned, we should skip processing the repo.
# It is either on a different commit or it has local changes.
if git_data != "":
logging.info(
"-- NOTE: Repo currently exists and appears to have local changes!"
)
logging.info("-- Skipping fetch!")
skip_repo = True
# Git Repos: STEP 2b ---------------------------------
# If we're not skipping, grab it.
if not skip_repo or force_it:
logging.info("## Fetching repo.")
# Using RunCmd for this one because the c.wait blocks incorrectly somehow.
cmd_string = "submodule update --init --recursive --progress"
if cache_path is not None:
cmd_string += " --reference " + cache_path
cmd_string += " " + required_repo
RunCmd('git', cmd_string, workingdir=my_workspace_path)
MuLogging.log_progress("Done.\n")
except RuntimeError as e:
logging.error("FAILED!\n")
logging.error("Failed to fetch required repository!\n")
log_lines(logging.ERROR, str(e))
# Now that we should have all of the required code,
# we're ready to build the environment and fetch the
# dependencies for this project.
MuLogging.log_progress("## Fetching all external dependencies...")
(build_env, shell_env) = minimum_env_init(my_workspace_path,
my_project_scope)
SelfDescribingEnvironment.UpdateDependencies(my_workspace_path,
my_project_scope)
MuLogging.log_progress("Done.\n")