def __init__(self, RootDir, OutputDir, YmlFile):
self.RootDirectory = None
self.YmlFilePathBase = None
self.YmlFilePathOut = None
self.OutputDirectory = None
self.MdFiles = list()
self.Repos = dict()
self.EncodingChecker = EncodingCheck()
#Convert RootDir to abs and confirm valid
if(RootDir is not None):
if(os.path.isabs(RootDir)):
self.RootDirectory = RootDir
else:
self.RootDirectory = os.path.join(os.path.abspath(os.getcwd()), RootDir)
self.RootDirectory = os.path.realpath(self.RootDirectory)
if(not os.path.isdir(self.RootDirectory)):
raise Exception("Invalid Path for RootDir: {0}".format(self.RootDirectory))
#Convert YmlFile to abs and confirm valid
if(YmlFile is not None):
if(os.path.isabs(YmlFile)):
self.YmlFilePathBase = YmlFile
else:
self.YmlFilePathBase = os.path.join(os.path.abspath(os.getcwd()), YmlFile)
self.YmlFilePathBase = os.path.realpath(self.YmlFilePathBase)
if(not os.path.isfile(self.YmlFilePathBase)):
raise Exception("Invalid Path for YmlFile: {0}".format(self.YmlFilePathBase))
self.YmlFilePathOut = os.path.join(os.path.dirname(self.YmlFilePathBase), "mkdocs.yml")
#Convert OutputDir to abs and then mkdir if necessary
if(OutputDir is not None):
if(os.path.isabs(OutputDir)):
self.OutputDirectory = OutputDir
else:
self.OutputDirectory = os.path.join(os.path.abspath(os.getcwd()), OutputDir)
self.OutputDirectory = os.path.realpath(self.OutputDirectory)
if(os.path.basename(self.OutputDirectory) != "docs"):
raise Exception("For mkdocs we only support output dir of docs. OutputDir: %s" % self.OutputDirectory)
self.OutputDirectory = os.path.join(OutputDir, DocBuild.DYNFOLDER) #set output to the dynamic folder
if(not os.path.isdir(self.OutputDirectory)):
logging.debug("Output directory doesn't exist. Making... {0}".format(self.OutputDirectory))
os.makedirs(self.OutputDirectory)
def __init__(self, RootDir, OutputDir, YmlFile):
self.RootDirectory = None
self.YmlFilePathBase = None
self.YmlFilePathOut = None
self.OutputDirectory = None
self.MdFiles = list()
self.Repos = dict()
self.EncodingChecker = EncodingCheck()
self.ExtraContents = dict()
# Convert RootDir to abs and confirm valid
if (RootDir is not None):
if (os.path.isabs(RootDir)):
self.RootDirectory = RootDir
else:
self.RootDirectory = os.path.join(os.path.abspath(os.getcwd()),
RootDir)
self.RootDirectory = os.path.realpath(self.RootDirectory)
if (not os.path.isdir(self.RootDirectory)):
raise Exception("Invalid Path for RootDir: {0}".format(
self.RootDirectory))
# Convert YmlFile to abs and confirm valid
if (YmlFile is not None):
if (os.path.isabs(YmlFile)):
self.YmlFilePathBase = YmlFile
else:
self.YmlFilePathBase = os.path.join(
os.path.abspath(os.getcwd()), YmlFile)
self.YmlFilePathBase = os.path.realpath(self.YmlFilePathBase)
if (not os.path.isfile(self.YmlFilePathBase)):
raise Exception("Invalid Path for YmlFile: {0}".format(
self.YmlFilePathBase))
self.YmlFilePathOut = os.path.join(
os.path.dirname(self.YmlFilePathBase), "mkdocs.yml")
# Convert OutputDir to abs and then mkdir if necessary
if (OutputDir is not None):
if (os.path.isabs(OutputDir)):
self.OutputDirectory = OutputDir
else:
self.OutputDirectory = os.path.join(
os.path.abspath(os.getcwd()), OutputDir)
self.OutputDirectory = os.path.realpath(self.OutputDirectory)
if (os.path.basename(self.OutputDirectory) != "docs"):
raise Exception(
"For mkdocs we only support output dir of docs. OutputDir: %s"
% self.OutputDirectory)
# set output to the dynamic folder
self.OutputDirectory = os.path.join(OutputDir, DocBuild.DYNFOLDER)
if (not os.path.isdir(self.OutputDirectory)):
logging.debug(
"Output directory doesn't exist. Making... {0}".format(
self.OutputDirectory))
os.makedirs(self.OutputDirectory)
# add all variables and functions here
self.ExtraContents["version"] = VERSION
self.ExtraContents["buildtime"] = str(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
self.ExtraContents["social"] = [{
"type":
'github',
"link":
'https://github.com/microsoft/mu'
}]
class DocBuild(object):
DYNFOLDER = "dyn"
#
# constructor that creates the DocBuild object
#
def __init__(self, RootDir, OutputDir, YmlFile):
self.RootDirectory = None
self.YmlFilePathBase = None
self.YmlFilePathOut = None
self.OutputDirectory = None
self.MdFiles = list()
self.Repos = dict()
self.EncodingChecker = EncodingCheck()
self.ExtraContents = dict()
# Convert RootDir to abs and confirm valid
if (RootDir is not None):
if (os.path.isabs(RootDir)):
self.RootDirectory = RootDir
else:
self.RootDirectory = os.path.join(os.path.abspath(os.getcwd()),
RootDir)
self.RootDirectory = os.path.realpath(self.RootDirectory)
if (not os.path.isdir(self.RootDirectory)):
raise Exception("Invalid Path for RootDir: {0}".format(
self.RootDirectory))
# Convert YmlFile to abs and confirm valid
if (YmlFile is not None):
if (os.path.isabs(YmlFile)):
self.YmlFilePathBase = YmlFile
else:
self.YmlFilePathBase = os.path.join(
os.path.abspath(os.getcwd()), YmlFile)
self.YmlFilePathBase = os.path.realpath(self.YmlFilePathBase)
if (not os.path.isfile(self.YmlFilePathBase)):
raise Exception("Invalid Path for YmlFile: {0}".format(
self.YmlFilePathBase))
self.YmlFilePathOut = os.path.join(
os.path.dirname(self.YmlFilePathBase), "mkdocs.yml")
# Convert OutputDir to abs and then mkdir if necessary
if (OutputDir is not None):
if (os.path.isabs(OutputDir)):
self.OutputDirectory = OutputDir
else:
self.OutputDirectory = os.path.join(
os.path.abspath(os.getcwd()), OutputDir)
self.OutputDirectory = os.path.realpath(self.OutputDirectory)
if (os.path.basename(self.OutputDirectory) != "docs"):
raise Exception(
"For mkdocs we only support output dir of docs. OutputDir: %s"
% self.OutputDirectory)
# set output to the dynamic folder
self.OutputDirectory = os.path.join(OutputDir, DocBuild.DYNFOLDER)
if (not os.path.isdir(self.OutputDirectory)):
logging.debug(
"Output directory doesn't exist. Making... {0}".format(
self.OutputDirectory))
os.makedirs(self.OutputDirectory)
# add all variables and functions here
self.ExtraContents["version"] = VERSION
self.ExtraContents["buildtime"] = str(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
self.ExtraContents["social"] = [{
"type":
'github',
"link":
'https://github.com/microsoft/mu'
}]
#
# delete the outputdirectory
#
def Clean(self):
retry = 1 # hack to make rmtree more successful
while True:
try:
if (self.OutputDirectory is not None
and os.path.isdir(self.OutputDirectory)):
shutil.rmtree(self.OutputDirectory)
except OSError:
if not retry:
# If we're out of retries, bail.
raise Exception("Failed to Clean dir {0}".format(
self.OutputDirectory))
time.sleep(2)
retry -= 1
continue
break
if (os.path.isfile(self.YmlFilePathOut)):
os.remove(self.YmlFilePathOut)
def _CleanChars(self, p: str) -> str:
'''mkdocs doesn't support all valid url chars so if they exist in the path they need to
be removed. Once such example is the . char. This is valid but not with mkdocs'''
# Bug filed here: https://github.com/mkdocs/mkdocs/issues/1924
d, e = os.path.splitext(
p) # split the path and extension so we can change path
d = d.replace(".", "")
d = d.replace(" ", "_")
return d + e
###########################################################################################
# Process functions - Start
###########################################################################################
#
# Process md files. Make folders in output and copy
#
# @apath - absolute path to md file
#
def _ProcessMarkdownFile(self, apath):
# Add relative path to list of md files
rpath = os.path.relpath(apath, self.RootDirectory)
rpath = self._CleanChars(rpath)
self.MdFiles.append(rpath)
logging.debug("md file found: {0}".format(rpath))
# Copy to output dir
s = apath
d = os.path.join(self.OutputDirectory, rpath)
os.makedirs(os.path.dirname(d), exist_ok=True)
shutil.copy2(s, d)
#
# Process Dec files. Collect info add add to list
#
# @apath - absolute path to dec file
#
def _ProcessEdk2DecFile(self, apath):
pass
def _ProcessImageFile(self, apath):
rpath = os.path.relpath(apath, self.RootDirectory)
logging.debug("image file found: {0}".format(rpath))
rpath = self._CleanChars(rpath)
# Copy to output dir
s = apath
d = os.path.join(self.OutputDirectory, rpath)
os.makedirs(os.path.dirname(d), exist_ok=True)
shutil.copy2(s, d)
#
# Process git repo. Collect git stats
#
# dirpath - absolute path for root of git directory
#
def _ProcessGitRepo(self, dirpath):
name = os.path.basename(dirpath)
u = GitSupport().get_url(dirpath)
b = GitSupport().get_branch(dirpath)
c = GitSupport().get_commit(dirpath)
d = GitSupport().get_date(dirpath, c)
cl = GitSupport().make_commit_url(c, u)
obj = {"url": u, "branch": b, "commit": c, "date": d, "commitlink": cl}
self.Repos[name] = obj
###########################################################################################
# Process functions - End
###########################################################################################
#
# walk the RootDirectory looking for md files
# #Copy the md files to OutputDirectory
#
# Also look for git repository roots
# #Collect more data about repos
#
#
def ProcessRootDir(self):
if (self.RootDirectory is None):
logging.debug("ProcessRootDir: No RootDirectory set.")
return
if self.OutputDirectory is None:
logging.debug("ProcessRootDir: No OutputDirectory set.")
return
for top, dirs, files in os.walk(self.RootDirectory):
for f in files:
if f.lower().endswith(".md"):
if (not self.EncodingChecker.TestMdEncodingOk(
os.path.join(top, f), "utf-8")):
logging.error(
"Ignore Invalid markdown file: {0}".format(
os.path.join(top, f)))
else:
self._ProcessMarkdownFile(os.path.join(top, f))
elif f.lower().endswith(".dec"):
self._ProcessEdk2DecFile(os.path.join(top, f))
elif f.lower().endswith(("_mu.gif", "_mu.png", "_mu.jpg")):
self._ProcessImageFile(os.path.join(top, f))
if (".git" in dirs):
# root of git repo
self._ProcessGitRepo(top)
return 0
def MakeYml(self):
f = open(self.YmlFilePathBase, "r")
f2 = open(self.YmlFilePathOut, 'w')
for l in f:
f2.write(l)
# now parse as yaml
f.seek(0)
# yaml.load(f)
# IMPORTANT NOTE: This call to "unsafe_load()" is only acceptable because we control the yml file being loaded.
# Unsafe load is required to support some configuration options for pymdownx.
if "extra" in yaml.unsafe_load(f):
raise Exception(
"extra: member not allowed in mkdocs_base.yml. Please add the contents to DocBuild constructor instead. "
)
f.close()
self.Yml = f2
def CloseYml(self):
if self.Yml is not None:
self.Yml.close()
with open(self.YmlFilePathOut, 'r') as a:
logging.debug("FINAL YML file")
logging.debug(a.read())
#
# Make yml nav output for the dynamic content
# Write it to the yml file
#
def MakeNav(self):
if self.YmlFilePathBase is None:
logging.debug("MakeNav: No YmlFilePathBase set.")
return
if self.RootDirectory is None:
logging.debug("MakeNav: No RootDirectory set.")
return
if self.OutputDirectory is None:
logging.debug("MakeNav: No OutputDirectory set.")
return
root = self._MakeNavTree()
root.Collapse()
navstring = root.GetNavYml("", " ")
self.Yml.write("\n - Code Repositories:")
self.Yml.write(navstring)
#
# Make yml config data for each repo
# Write it to the yml file
#
def MakeRepoInfo(self):
if self.Yml is None:
logging.debug("MakeRepoInfo: No open Yml file.")
return
if self.RootDirectory is None:
logging.debug("MakeRepoInfo: No RootDirectory set.")
return
if self.OutputDirectory is None:
logging.debug("MakeRepoInfo: No OutputDirectory set.")
return
logging.debug(str(self.Repos))
self.ExtraContents.update(self.Repos)
def WriteExtra(self):
if self.Yml is None:
logging.debug("WriteExtra: No open Yml file.")
return
self.Yml.write("\n#AutoGenerated based on ExtraContents in DocBuild\n")
yaml.dump({"extra": self.ExtraContents},
self.Yml,
default_flow_style=False)
#
# Internal function
def _MakeNavTree(self):
root = NavTree()
for a in self.MdFiles:
string1 = a.replace(os.sep, "/")
string2 = string1.rpartition(".")[0] # remove the md extension
# this is intentionally not os.sep
root.AddToTree(string2, DocBuild.DYNFOLDER + "/" + string1)
logging.debug(root)
return root
class DocBuild(object):
DYNFOLDER = "dyn"
#
# constructor that creates the DocBuild object
#
def __init__(self, RootDir, OutputDir, YmlFile):
self.RootDirectory = None
self.YmlFilePathBase = None
self.YmlFilePathOut = None
self.OutputDirectory = None
self.MdFiles = list()
self.Repos = dict()
self.EncodingChecker = EncodingCheck()
#Convert RootDir to abs and confirm valid
if(RootDir is not None):
if(os.path.isabs(RootDir)):
self.RootDirectory = RootDir
else:
self.RootDirectory = os.path.join(os.path.abspath(os.getcwd()), RootDir)
self.RootDirectory = os.path.realpath(self.RootDirectory)
if(not os.path.isdir(self.RootDirectory)):
raise Exception("Invalid Path for RootDir: {0}".format(self.RootDirectory))
#Convert YmlFile to abs and confirm valid
if(YmlFile is not None):
if(os.path.isabs(YmlFile)):
self.YmlFilePathBase = YmlFile
else:
self.YmlFilePathBase = os.path.join(os.path.abspath(os.getcwd()), YmlFile)
self.YmlFilePathBase = os.path.realpath(self.YmlFilePathBase)
if(not os.path.isfile(self.YmlFilePathBase)):
raise Exception("Invalid Path for YmlFile: {0}".format(self.YmlFilePathBase))
self.YmlFilePathOut = os.path.join(os.path.dirname(self.YmlFilePathBase), "mkdocs.yml")
#Convert OutputDir to abs and then mkdir if necessary
if(OutputDir is not None):
if(os.path.isabs(OutputDir)):
self.OutputDirectory = OutputDir
else:
self.OutputDirectory = os.path.join(os.path.abspath(os.getcwd()), OutputDir)
self.OutputDirectory = os.path.realpath(self.OutputDirectory)
if(os.path.basename(self.OutputDirectory) != "docs"):
raise Exception("For mkdocs we only support output dir of docs. OutputDir: %s" % self.OutputDirectory)
self.OutputDirectory = os.path.join(OutputDir, DocBuild.DYNFOLDER) #set output to the dynamic folder
if(not os.path.isdir(self.OutputDirectory)):
logging.debug("Output directory doesn't exist. Making... {0}".format(self.OutputDirectory))
os.makedirs(self.OutputDirectory)
#
# delete the outputdirectory
#
def Clean(self):
retry = 1 #hack to make rmtree more successful
while True:
try:
if(self.OutputDirectory is not None and os.path.isdir(self.OutputDirectory)):
shutil.rmtree(self.OutputDirectory)
except OSError:
if not retry:
# If we're out of retries, bail.
raise Exception("Failed to Clean dir {0}".format(self.OutputDirectory))
time.sleep(2)
retry -= 1
continue
break
if(os.path.isfile(self.YmlFilePathOut)):
os.remove(self.YmlFilePathOut)
###########################################################################################
## Process functions - Start
###########################################################################################
#
# Process md files. Make folders in output and copy
#
# @apath - absolute path to md file
#
def _ProcessMarkdownFile(self, apath):
# Add relative path to list of md files
rpath = os.path.relpath(apath, self.RootDirectory)
self.MdFiles.append(rpath)
logging.debug("md file found: {0}".format(rpath))
#Copy to output dir
s = apath
d = os.path.join(self.OutputDirectory, rpath)
os.makedirs(os.path.dirname(d), exist_ok=True)
shutil.copy2(s, d)
#
# Process Dec files. Collect info add add to list
#
# @apath - absolute path to dec file
#
def _ProcessEdk2DecFile(self, apath):
pass
#
# Process git repo. Collect git stats
#
# dirpath - absolute path for root of git directory
#
def _ProcessGitRepo(self, dirpath):
name = os.path.basename(dirpath)
u = GitSupport().get_url(dirpath)
b = GitSupport().get_branch(dirpath)
c = GitSupport().get_commit(dirpath)
d = GitSupport().get_date(dirpath, c)
cl= GitSupport().make_commit_url(c,u)
obj = { "url": u, "branch": b, "commit": c, "date": d, "commitlink": cl}
self.Repos[name] = obj
###########################################################################################
## Process functions - End
###########################################################################################
#
# walk the RootDirectory looking for md files
# #Copy the md files to OutputDirectory
#
# Also look for git repository roots
# #Collect more data about repos
#
#
def ProcessRootDir(self):
if(self.RootDirectory is None):
logging.debug("ProcessRootDir: No RootDirectory set.")
return
if self.OutputDirectory is None:
logging.debug("ProcessRootDir: No OutputDirectory set.")
return
for top, dirs, files in os.walk(self.RootDirectory):
for f in files:
if f.lower().endswith(".md"):
if( not self.EncodingChecker.TestMdEncodingOk(os.path.join(top, f))):
logging.error("Ignore Invalid markdown file: {0}".format(os.path.join(top, f)))
else:
self._ProcessMarkdownFile(os.path.join(top, f))
elif f.lower().endswith(".dec"):
self._ProcessEdk2DecFile(os.path.join(top, f))
if(".git" in dirs):
#root of git repo
self._ProcessGitRepo(top)
return 0
def MakeYml(self):
f = open(self.YmlFilePathBase, "r")
f2 = open(self.YmlFilePathOut, 'w')
for l in f:
f2.write(l)
f.close()
self.Yml = f2
def CloseYml(self):
if self.Yml is not None:
self.Yml.close()
with open(self.YmlFilePathOut, 'r') as a:
logging.debug("FINAL YML file")
logging.debug(a.read())
#
# Make yml nav output for the dynamic content
# Write it to the yml file
#
def MakeNav(self):
if self.YmlFilePathBase is None:
logging.debug("MakeNav: No YmlFilePathBase set.")
return
if self.RootDirectory is None:
logging.debug("MakeNav: No RootDirectory set.")
return
if self.OutputDirectory is None:
logging.debug("MakeNav: No OutputDirectory set.")
return
root = self._MakeNavTree()
root.Collapse()
navstring = root.GetNavYml("", " ")
self.Yml.write("\n - Code Repositories:")
self.Yml.write(navstring)
#
# Make yml config data for each repo
# Write it to the yml file
#
def MakeRepoInfo(self):
if self.Yml is None:
logging.debug("MakeRepoInfo: No open Yml file.")
return
if self.RootDirectory is None:
logging.debug("MakeRepoInfo: No RootDirectory set.")
return
if self.OutputDirectory is None:
logging.debug("MakeRepoInfo: No OutputDirectory set.")
return
self.Yml.write("\nextra:\n")
for (k,v) in self.Repos.items():
logging.debug(k + str(v))
self.Yml.write(" " + k + ":\n")
self.Yml.write(" url: " + v["url"] + "\n")
self.Yml.write(" commit: " + v["commit"] + "\n")
self.Yml.write(" branch: " + v["branch"] + "\n")
self.Yml.write(" commitlink: " + v["commitlink"] + "\n")
self.Yml.write(" date: " + v["date"] + "\n")
#
# Internal function
def _MakeNavTree(self):
root = NavTree()
for a in self.MdFiles:
string1 = a.replace(os.sep, "/")
string2 = string1.partition(".")[0]
root.AddToTree(string2, DocBuild.DYNFOLDER+ "/"+ string1)
logging.debug(root)
return root