def writeReadme():
""" Extract readme file from readme path in config.
If not specified file containing <readme> in scriptsPath with be chosen"""
conf = Config()
scriptsPath = conf.get("scriptsPath")
snakeroot = conf.snakeroot
try:
filename_readme = conf.get("readmePath") #### should be .md file
except AttributeError as e:
filename_readme = ""
onlyfiles = [
f for f in os.listdir(snakeroot)
if os.path.isfile(os.path.join(snakeroot, f))
]
for f in onlyfiles:
if ("readme" in f) and f.endswith(".md"):
filename_readme = f
filename_readme = filename_readme.replace(".md", ".html")
readmeString = '<li><a href="javascript:navigate(' + " '{}'".format(
filename_readme) + ');">Readme</a></li> '
readmeIframeString = '<iframe id="Iframe" src="' + filename_readme + '" width=100% height=95% ></iframe> '
readmeFilename = ' "{}" '.format(filename_readme)
return readmeString, readmeIframeString, readmeFilename
def insertPlaceholders(dest, source):
"""
Infer placeholders' substitutions.
:param dest: string to replace placeholders in
:param source: file; from its path we infer the placeholders values
:return: dest with replaced placeholders
"""
path = pathlib.Path(source) #get the path to the file
conf = Config()
processedDataPath = conf.get("processedDataPath")
PD = pathlib.Path(processedDataPath)
PP = path.parts[-2]
dest = dest.replace("{wbPD}", str(PD))
dest = dest.replace("{wbPP}", str(PP))
if len(path.parts) <= 2 and bool(re.search('{wbP(D_P*)?}', source)):
print("If using placeholders please make sure you have the right",
" directory structure.")
if len(path.parts) > 2:
P = path.parts[-3]
dest = dest.replace("{wbPD_P}", str(PD / P))
dest = dest.replace("{wbPD_PP}", str(PD / P / PP))
dest = dest.replace("{wbP}", str(P))
return dest
def getRecentMenu():
"""
Support recently edited files list to the HTML web output.
:return: HTML string: "Recently viewed" menu contents
"""
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
rFiles = sorted([
join(htmlOutputPath, f)
for f in listdir(htmlOutputPath) if isfile(join(htmlOutputPath, f))
],
key=os.path.getmtime,
reverse=True)
## delete all files containing the word "index from html menu "
rFiles = [f for f in rFiles if "index" not in f]
rFiles = rFiles[:10]
# open recent Files in new tab
menuString = ""
for f in rFiles:
fo = pathlib.PurePath(f).name
# Open in a new tab
#menuString += ('<p><a href='+ fo + ' target="_blank">' + fo.replace('_', ' ').replace('.html', '') +
# '</a></p>\n')
# Open in same tab
menuString += ('<p><a href="javascript:navigate(\'' + fo + '\');" >' +
fo.replace('_', ' ').replace('.html', '') +
'</a></p>\n')
return menuString
def writeDepSVG():
""" Search for rule graph. If path not specified in config, take default dep.svg in snakeroot path"""
conf = Config()
scriptsPath = conf.get("scriptsPath")
htmlOutputPath = conf.get("htmlOutputPath")
snakeroot = conf.snakeroot
foldername = snakeroot.split("/")[-1]
try:
filename_SVG = conf.get("ruleGraphPath") #### should be .md file
except AttributeError as e:
### try with default name "dep.svg"
if os.path.isfile(os.path.join(htmlOutputPath, "dep.svg")):
filename_SVG = "dep.svg"
else:
### search for files containing "svg" and foldername
filename_SVG = ""
onlyfiles = [
f for f in os.listdir(htmlOutputPath)
if os.path.isfile(os.path.join(htmlOutputPath, f))
]
for f in onlyfiles:
if (foldername in f) and f.endswith(".svg"):
filename_SVG = f
svgString = '<li><a href="javascript:navigate(' + "'{}'".format(
filename_SVG) + ');">Dependency</a></li>'
return svgString
def autolink(config):
conf = Config()
scriptsPath = conf.get("scriptsPath")
S = Path(scriptsPath)
tasks = load(open(config))
for filename in glob.iglob(scriptsPath + '/**/*.ln.R', recursive=True):
os.remove(filename)
for task in tasks:
print(task)
if task['dst'] is None:
continue
if task['src'] is None:
continue
for dst in task['dst']:
if dst is None:
continue
if not os.path.exists(str(S / dst)):
os.makedirs(str(S / dst))
for src in task['src']:
if src is None:
continue
link(str(S / Path(src)), str(S / Path(dst) / Path(src).stem) + '.ln.R')
def writeIndexRule(wbData, mdData, file, ignoreMD=False, dump=False):
"""
Write the rule of mapping the R and md wbData to the index.html.
:param wbRRows: info dict parsed from R wB files
:param wbMDrows: info dict parsed from MD wB files
:param file: file to print the index rule to
"""
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
for r in wbData:
writeRule(r, file, dump)
if not ignoreMD:
for r in mdData:
writeMdRule(r, file)
input, output, graphPath, _ = wbuild.createIndex.createIndexRule(
wbData=wbData, mdData=mdData)
# write rule
file.write('\n')
file.write('rule Index:\n')
file.write(' input: \n "' + '",\n "'.join(input) + '"\n')
file.write(' output: \n')
file.write(' index = "' + output + '", \n')
file.write(' graph = "' + graphPath + '" \n')
# file.write(' script: ".wBuild/createIndex.py"\n')
file.write(' run:\n')
file.write(' import wbuild.createIndex\n')
file.write(' wbuild.createIndex.ci()\n')
file.write(
' shell("snakemake --rulegraph | dot -Tsvg -Grankdir=RL > {output.graph}")\n'
)
file.write('\n')
def writeDependencyFile():
"""
Entry point for writing .wBuild.depend.
"""
# if not wbuildVersionIsCurrent():
# print(bcolors.WARNING + "Version of the project's static .wBuild lib is not the same as the dynamically loaded "
# "wBuild"
# "version. It is strongly recommended to update .wBuild lib using \'wbuild update\'; "
# "otherwise, the consistency of the build can not be guaranteed." + bcolors.ENDC)
logger.info("Structuring dependencies...")
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
logger.debug("Loaded config.\n html output path (key htmlOutputPath): " +
htmlOutputPath + "\n")
scriptsPath = conf.get("scriptsPath")
readmePath = conf.get("readmePath")
wbData = parseWBInfosFromRFiles(script_dir=scriptsPath,
htmlPath=htmlOutputPath)
mdData = parseMDFiles(script_dir=scriptsPath,
htmlPath=htmlOutputPath,
readmePath=readmePath)
dependFile = tempfile.NamedTemporaryFile('w', delete=False)
with dependFile as f: #start off with the header
f.write('######\n')
f.write('#This is a autogenerated snakemake file by wBuild\n')
f.write('#wBuild by Leonhard Wachutka\n')
f.write('######\n')
# write build index rule
writeIndexRule(wbData, mdData, f)
logger.info("Dependencies file generated at: {}\n".format(dependFile.name))
return dependFile.name
def writeIndexHTMLMenu(scriptsPath=None, index_name=None):
"""
Scan for files involved in the current HTML rendering and fill the HTML quick access toolbar correspondingly
"""
conf = Config()
if scriptsPath is None:
scriptsPath = conf.get("scriptsPath")
htmlOutputPath = conf.get("htmlOutputPath")
pageTitle = conf.get("projectTitle")
snakeroot = conf.snakeroot
wbData = parseWBInfosFromRFiles(script_dir=scriptsPath,
htmlPath=htmlOutputPath)
mdData = parseMDFiles(script_dir=scriptsPath, htmlPath=htmlOutputPath)
wbData += mdData
temp = []
# for all of the scanned files, collect their paths
for r in wbData:
# this is needed so the relative path to "../wbuild/Snakefile" is not
# part of the html sub menu
r['file'] = removeFilePrefix(r['file'], snakeroot)
temp.append(pathlib.PurePath(r['file']).parts[1])
menuString = ""
for top in sorted(set(temp)):
menuString += (
'<li class="dropdown">\n' +
# write the current directory's name to the main ("top") toolbar tab
' <a href="#" class="dropdown-toggle" data-toggle="dropdown" ' +
'role="button" aria-haspopup="true" aria-expanded="false">' + top +
'<span class="caret"></span></a>\n'
' <ul class="dropdown-menu multi-level" role="menu">\n' +
# write sub-directories to the dropdown list of the "top" tabs
writeSubMenu(top, wbData, 2) + ' </ul>\n' + '</li>\n')
_, output, graphPath, readmePath = createIndexRule(scriptsPath, index_name)
readmeString, readmeIframeString, readmeFilename = writeReadme(readmePath)
depSVGString = writeDepSVG(graphPath)
# fill the HTML template with the constructed tag structure
wbuildPath = pathlib.Path(wbuild.__file__).parent
template = open(str(wbuildPath / "html/template.html")).read()
template = Template(template).substitute(menu=menuString,
title=pageTitle,
rf=getRecentMenu(),
readme=readmeString,
readmeIframe=readmeIframeString,
readmeFilename=readmeFilename,
depSVG=depSVGString)
f = open(output, 'w')
f.write(template)
f.close()
def createIndexName(scriptsPath, default=None):
if default is not None:
return default
name = ""
conf = Config()
indexWithFolderName = conf.get("indexWithFolderName")
if indexWithFolderName:
abs_path = str(os.path.abspath(scriptsPath))
name = abs_path.split("/")[-2]
return name
def ci():
writeIndexHTMLMenu()
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
libDir = htmlOutputPath + "/lib"
if os.path.exists(libDir):
shutil.rmtree(libDir)
wbuildPath = pathlib.Path(wbuild.__file__).parent
shutil.copytree(str(wbuildPath) + "/html/lib", libDir)
def writeWBParseDependencyFile(filename):
"""
Entry point for writing .wBuild.depend. for the wbParseFunction in R
"""
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
wbData = parseWBInfosFromRFile(filename=filename, htmlPath=htmlOutputPath)
with open('.wBuild.depend', 'w') as f: #start off with the header
f.write('######\n')
f.write('#This is a autogenerated snakemake file by wBuild\n')
f.write('#wBuild by Leonhard Wachutka\n')
f.write('######\n')
# write rules
for r in wbData:
writeRule(r, f, True)
writeIndexRule(wbData, list(), f)
logger.info("Dependencies file generated.\n")
def ci(scriptsPath=None, index_name=None):
"""
Write HTML index file
:param scriptsPath: relative scripts path. If not specified, use the default scripts path from the config.
:param index_name: prefix of the index file name `<index_name>_index.html`
"""
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
writeIndexHTMLMenu(scriptsPath, index_name)
libDir = htmlOutputPath + "/lib"
# if os.path.exists(libDir):
# shutil.rmtree(libDir)
if not os.path.exists(libDir):
wbuildPath = pathlib.Path(wbuild.__file__).parent
shutil.copytree(str(wbuildPath) + "/html/lib", libDir)
def writeIndexRule(wbRRows, wbMDrows, file):
"""
Write the rule of mapping the R and md wbData to the index.html.
:param wbRRows: info dict parsed from R wB files
:param wbMDrows: info dict parsed from MD wB files
:param file: file to print the index rule to
"""
inputFiles = []
for r in wbRRows:
#ignore if the file is script or noindex
if getYamlParam(r, 'type') == 'script' or getYamlParam(
r, 'type') == 'noindex':
continue
inputFiles.append(r['outputFile'])
for r in wbMDrows:
inputFiles.append(r['outputFile'])
conf = Config()
htmlOutputPath = conf.get("htmlOutputPath")
try:
filename_index = conf.get("htmlIndex")
except AttributeError as e:
filename_index = "index.html"
try:
indexWithFolderName = conf.get("indexWithFolderName")
except:
indexWithFolderName = False
if indexWithFolderName:
scriptsPath = conf.get("scriptsPath")
abs_path = str(os.path.abspath(scriptsPath))
name = abs_path.split("/")[-2]
filename_index = name + "_" + filename_index
file.write('\n')
file.write('rule Index:\n')
file.write(' input: \n "' + '",\n "'.join(inputFiles) +
'"\n')
file.write(' output: "' + htmlOutputPath + '/' + filename_index +
'" \n')
# file.write(' script: ".wBuild/createIndex.py"\n')
file.write(' run:\n')
file.write(' import wbuild.createIndex\n')
file.write(' wbuild.createIndex.ci()\n')
file.write('\n')
def createIndexRule(scriptsPath=None,
index_name=None,
wbData=None,
mdData=None):
"""
Create the input and output files necessary for an HTML index rule.
:param scriptsPath: relative scripts path. If not specified, use the default scripts path from the config.
:param index_name: prefix of the index file name `<index_name>_index.html`
:param wbData: wBuild rule data from parsed R scripts
:param mdData: wBuild rule data from parsed markdown files
:return:
+ **inputFiles** - list of output HTML files from the `scriptsPath`, comprising the input of the index rule
+ **indexPath** - index html file name, equates to the output
+ **graphPath** - dependency graph image file name for HTML template (graph is needs to be written to this file)
+ **readmePath** - readme HTML name for HTML template (takes readme from the `scriptsPath`)
"""
conf = Config()
if scriptsPath is None or scriptsPath == conf.get("scriptsPath"):
readmePath = conf.get("readmePath")
scriptsPath = conf.get("scriptsPath")
else: # find readme file
readmePath = findFirstFile(scriptsPath, ".*readme.*", "md", re.I)
if readmePath is None:
readmePath = scriptsPath + "/readme.md"
open(readmePath, "a").close()
htmlOutputPath = conf.get("htmlOutputPath")
htmlIndex = conf.get("htmlIndex")
index_name = createIndexName(scriptsPath, index_name)
# gather index input files
inputFiles = []
if wbData is None:
wbData = parseWBInfosFromRFiles(script_dir=scriptsPath,
htmlPath=htmlOutputPath)
if mdData is None:
mdData = parseMDFiles(script_dir=scriptsPath,
htmlPath=htmlOutputPath,
readmePath=readmePath)
for r in wbData:
# ignore if the file is script or noindex
if getYamlParam(r, 'type') == 'script' or getYamlParam(
r, 'type') == 'noindex':
continue
inputFiles.append(r['outputFile'])
for r in mdData:
inputFiles.append(r['outputFile'])
if len(index_name) > 0:
index_name = index_name + "_" # separator for distinct index_name
indexPath = "/".join([htmlOutputPath, index_name + htmlIndex])
graphPath = "/".join([htmlOutputPath, index_name + "dep.svg"])
# readme html
readmePath = htmlOutputPath + "/" + pathsepsToUnderscore(
os.path.splitext(readmePath)[0]) + ".html"
return inputFiles, indexPath, graphPath, readmePath
