def index_exe(self):
""" Which source file to use for index page
"""
if len(self.index_exes) == 1:
return self.index_exes[0]
else:
return os.path.join(get_sk_exe_dir(), "workflow", "notebook_rules",
'index.Rmd')
def detect_snakemake_progress(line, quiet=False):
ntbd_match = re.match(".*Nothing to be done..*", line)
job_match = re.match("^Job .*: (.*)$",line)
if ntbd_match:
warn("sk: Nothing to be done")
elif job_match:
# sanitize system index.Rmd path
job_msg = job_match.groups()[0].replace(get_sk_exe_dir(),"system's ")
if job_msg[0] ==' ' and quiet:
return
warn("sk: " + job_msg)
def main():
yaml = YAML(typ="rt")
skconfig = ScikickConfig()
# get tab strucutre
tabs = get_tabs(skconfig)
site_yaml_files = skconfig.get_site_yaml_files()
giturl = git_repo_url()
if giturl != '.':
# get git repo url
nav_more = {"text": "More", \
"menu": [{"text" : "Git Repository", "href" : giturl}]}
else:
nav_more = {}
# translating the desired layout (get_tabs) to _site.yml format
for site_yaml_file in site_yaml_files:
nav_left = list()
# path from the site yaml to the output root
path_to_root = relpath(join(skconfig.report_dir,'out_md'),start=dirname(site_yaml_file))
for tab, items in tabs.items():
human_text = clean_name(tab)
# if first value is the key, this is a file
tabisfile = basename(items[0]) == tab
if tabisfile:
path_from_site_to_html = join(path_to_root,items[0])
this_item = {"text": human_text, "href": "%s.html" % path_from_site_to_html}
else:
this_item = {"text": human_text, "menu":[]}
for item in items:
path_from_site_to_html = join(path_to_root,item)
sub_item = {"text": clean_name(basename(item)), "href": "%s.html" % path_from_site_to_html}
this_item['menu'].append(sub_item)
nav_left.append(this_item)
site_yaml= { "navbar": {"title": clean_name(basename(getcwd())), \
"left": nav_left}}
if nav_more != {}:
site_yaml["navbar"]["right"] = [nav_more]
if 'output' in skconfig.config.keys():
output_yaml = skconfig.config
else:
# TODO merge with scikick.yml
output_yaml = yaml.load(open(join(get_sk_exe_dir(),"workflow/site_rules/default_output.yml"),"r"))
output_yaml['output']['rmarkdown::html_document']['pandoc_args'] = '--resource-path=.:' + path_to_root + '/../../'
site_yaml.update(output_yaml)
yaml.indent(sequence=4, mapping=4, offset=0)
yaml.dump(site_yaml, open(site_yaml_file, "w"))
def rm(files, deps):
""" Delete files and dependencies from them
files - page file list
deps - dependency file list
"""
if deps is None:
deps = list()
ymli = yaml_in(need_pages=True)
for fname in files:
# check if rmd included
if fname not in ymli['analysis'].keys():
warn(f"sk: Warning: File {fname} not included")
continue
# delete script entry if no dependencies specified
if len(deps) == 0:
del ymli['analysis'][fname]
warn(f"sk: {fname} removed")
# Check if fname was a dependency for other scripts
for val in ymli['analysis'].values():
if val is not None:
if fname in val:
warn(
f"sk: Warning: {fname} is still a dependency of other scripts"
)
warn(
f"sk: Use sk del -d {fname} <script> to change this"
)
# delete only deps if deps specified
else:
if ymli['analysis'][fname] is None:
warn(f"sk: Warning: File {fname} has no dependencies")
continue
for dep in deps:
if dep in ymli['analysis'][fname]:
ymli['analysis'][fname].remove(dep)
warn(f"sk: dependency {dep} removed from {fname}")
else:
warn(f"sk: no dependency {dep} found for {fname}")
if len((ymli['analysis'][fname])) == 0:
ymli['analysis'][fname] = None
if os.path.splitext(os.path.basename(fname))[0] == "index":
index_list = get_indexes(ymli)
if len(index_list) == 0:
warn(f"sk: Using system template index.Rmd as homepage")
os.utime(
os.path.join(get_sk_exe_dir(), "workflow",
"notebook_rules", "index.Rmd"), None)
elif len(index_list) == 1:
os.utime(index_list[0], None)
yaml_dump(ymli)
def init_yaml():
"""Create an initial scikick.yml config file and update the project scikick
version"""
template_yaml_path = os.path.join(get_sk_exe_dir(), 'usr/scikick.yml')
project_dir = os.getcwd()
proj_yaml_path = os.path.join(project_dir, "scikick.yml")
yml_loader = ruamel.yaml.YAML()
check_requirements()
if os.path.exists(proj_yaml_path):
warn("sk: File scikick.yml already exists, adding current scikick version")
yml_out = yml_loader.load(open(proj_yaml_path, "r"))
else:
warn("sk: Importing template analysis configuration file")
yml_out = yml_loader.load(open(template_yaml_path, "r"))
yml_out = add_version_info(yml_out)
warn("sk: Writing to scikick.yml")
yml_loader.dump(yml_out, open(proj_yaml_path,"w"))
def run_demo_stage1():
print_demo("A demo project will be used to demonstrate some features of scikick.")
print("")
print_demo("---------- Starting a New Project ----------")
print_demo("A new scikick project can be initialized with sk init which will:")
print_demo("check some software dependencies,")
print_demo("add the scikick.yml config file (-y),")
print_demo("and make some useful directories (-d).")
print_exec("sk init -yd")
# copy the files
demo_templatedir = os.path.join(get_sk_exe_dir(), "demo")
for f in ["generate.Rmd", "PCA.Rmd", "analysis_config.txt", \
"PC_score_statistics.Rmd", "index.Rmd"]:
shutil.copy(os.path.join(demo_templatedir, f), os.path.join("code", f))
print_demo("Demo project has been initialized.")
print_demo("Run sk init --demo again to continue.")
def init_git():
"""Add certain entries to .gitignore"""
usr_dir = os.path.join(get_sk_exe_dir(), 'usr')
gitignore = ".gitignore"
anything_appended = False
if copy_file(os.path.join(usr_dir, gitignore), \
os.path.join(gitignore)):
warn("sk: File .gitignore created")
else:
existing_gitignore = open(gitignore, 'r').readlines()
template_gitignore = open(os.path.join(usr_dir, \
gitignore), 'r').readlines()
append_existing = open(gitignore, "a")
for ignore_entry in template_gitignore:
if ignore_entry.strip() not in map(str.strip, \
existing_gitignore):
append_existing.writelines(ignore_entry)
warn("sk: Added \'%s\' to .gitignore" % \
ignore_entry.rstrip())
anything_appended = True
append_existing.close()
if not anything_appended:
warn("sk: .gitignore already has all the required entries")
def add_check(fname, ymli, force, deps):
"""Performs a check if fname can be added to scikick.yml as a key"""
pages = ymli["analysis"].keys()
if fname in pages:
warn(f"sk: Found existing entry for {fname}")
return False
# filenames cannot currently have wildcard symbols
if True in [i in fname for i in wildcard_symbols]:
warn(
f"sk: Error: Filename ({fname}) cannot have wildcard symbols ({' '.join(wildcard_symbols)})"
)
return False
# check if the file extension is supported
fext = os.path.splitext(fname)[-1]
if fext.lower() == ".ipynb":
warn(
"sk: Warning: .ipynb use in Scikick is experimental and requires installation of jupyter"
)
f_exe_support = fext.lower() in [x.lower() for x in supported_extensions]
if not f_exe_support:
extension_list_str = ', '.join(supported_extensions)
warn("sk: Error: Only %s files can be added as pages (%s)" % \
(extension_list_str, fname))
return False
# error if the directory doesn't exist
dirname = os.path.dirname(fname)
if dirname != "":
if not os.path.isdir(dirname):
reterr(f"sk: Error: Directory {dirname} does not exist.")
# create the file if it doesn't exist
if not os.path.isfile(fname):
warn(f"sk: Warning: File {fname} doesn't exist")
warn(f"sk: Creating new file {fname}")
open(fname, "a").close()
if fname not in pages:
# Check for other files with same basename (and therefore same md output file)
fname_base = os.path.splitext(fname)[0]
page_basenames = map(lambda x: os.path.splitext(x)[0], pages)
page_basename_exists = fname_base in page_basenames
if page_basename_exists:
warn(
f"sk: Error: Page {fname_base} is already to be compiled from another file."
)
return False
# check for "index.Rmd"s
index_list = get_indexes(ymli)
if os.path.splitext(os.path.basename(fname))[0] == "index":
if len(index_list) == 0:
warn(
f"sk: An index file {fname} has been added and will be used as the homepage"
)
# touch the added index file to ensure execution
os.utime(fname, None)
elif len(index_list) == 1:
if not force:
reterr(f"sk: Error: An index file {index_list[0]} already exists\n" + \
"sk: Error: An additional one can be added, but neither will be used as a homepage\n" + \
f"sk: Error: To persist, use 'sk add --force {fname}'")
else:
warn(
f"sk: Warning: A redundant index file has been added\n"
+
"sk: Warning: Neither of the added index files will be used as a homepage"
)
os.utime(
os.path.join(get_sk_exe_dir(), "workflow",
"notebook_rules", "index.Rmd"), None)
elif len(index_list) > 1:
warn(
f"sk: Warning: A redundant index file has been added\n" +
"sk: Warning: Neither of the added index files will be used as a homepage"
)
return True
def run_snakemake(snakefile=get_sk_snakefile(), workdir=os.getcwd(), \
verbose=False, dryrun=False, snakeargs=None, rmds=[], quiet=False):
"""Run snakemake with specified arguments
snakefile -- string (path to the main snakefile)
workdir -- string
verbose -- bool
dryrun -- bool
snakeargs -- list (list of additional arguments to snakemake)
rmds -- string rmd who's output should be targetted
"""
exe_dir = get_sk_exe_dir()
loghandler = os.path.join(exe_dir, 'workflow/loghandler.py')
skconf = ScikickConfig()
yml = skconf.config
# logfile created by snakemake
snake_logfile=""
### Construct call to snakemake
# before 'snakemake'
env_vars = f'SINGULARITY_BINDPATH="{exe_dir}"'
# after 'snakemake'
snakemake_args = ""
snakemake_args += f" --snakefile {snakefile}"
snakemake_args += f" --directory '{workdir}'"
snakemake_args += " --cores 1"
snakemake_args += f" --log-handler-script {loghandler}"
# Translate Rmd script to HTML target
# TODO - factor out
# TODO - move this to sk_run as an additional snake_arg
# to reduce skconfig read ins
target_arg = ""
if len(rmds) > 0:
for rmd in rmds:
# Try to add rmd if it is not in scikick.yml
if yml['analysis'] is None:
rmd_found = False
elif rmd not in yml["analysis"].keys():
rmd_found = False
else:
rmd_found = True
if not rmd_found:
warn(f"sk: Warning: {rmd} was not found in scikick.yml")
if os.path.exists(rmd):
scikick.yaml.add([rmd])
# Must reload or modify the yml since it changed
skconf = ScikickConfig()
yml = skconf.config
else:
warn(f"sk: Warning: Will not try to add non-existing file {rmd}")
return 0
# Set target. Index file is treated differently
index_rmds = get_indexes(yml)
if (len(index_rmds) == 1) and (index_rmds[0] == rmd):
target_arg += " " + os.path.join(yml["reportdir"], \
"out_html", "index.html")
else:
target_arg += " " + os.path.join(yml["reportdir"], \
"out_html", os.path.splitext(rmd)[0] + ".html")
# set more snakemake arguments
if dryrun:
snakemake_args += " --dry-run"
# Add user defined snakemake arguments
if snakeargs is not None:
snakemake_args += f" {snakeargs}"
if 'snakemake_args' in yml.keys() and yml['snakemake_args'] is not None:
warn("sk: Warning: snakemake_args is deprecated")
snakemake_args += f" {' '.join(yml['snakemake_args'])}"
# add the implied targets (html of Rmd)
snakemake_args += f" {target_arg}"
# Check for a user defined snakefile (imported by scikick Snakefile)
user_snakefile = os.path.join(os.getcwd(), "Snakefile")
if os.path.isfile(user_snakefile):
warn("sk: Including Snakefile found in project directory")
### Execution
if verbose:
warn("sk: Starting snakemake")
cmd = f"{env_vars} snakemake {snakemake_args}"
print(cmd)
sys.exit(subprocess.call(cmd, shell=True))
else:
snake_p = subprocess.Popen(f"snakemake {snakemake_args}", \
shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logs=[]
sm_err = 0
page_err = 0
while True:
line = snake_p.stdout.readline().decode('utf-8')
if not line:
break
# Capture logs
logs.append(line)
# Report progress
detect_snakemake_progress(line,quiet)
page_err = detect_page_error(line) or page_err
# In case of snakemake error start writing stderr
sm_err += detect_snakemake_error(line)
if sm_err:
sys.stderr.write(line)
logfile_name_match = re.match("^SK INTERNAL: logfile (.*)$", line)
if logfile_name_match is not None:
# Use the first found match
if snake_logfile == "":
snake_logfile = logfile_name_match.groups()[0]
snake_p.wait()
if snake_p.returncode != 0:
if not page_err:
warn("sk: Error: Snakemake returned a non-zero return code")
if not sm_err:
warn("sk: Warning: scikick was unable to find snakemake error in logs, dumping stderr...")
for line in logs:
sys.stderr.write(line)
return snake_p.returncode
else:
if not os.path.exists(skconf.homepage):
warn(f"sk: Warning: Expected homepage {skconf.homepage} is missing")
if snake_logfile != "":
rellog=os.path.relpath(snake_logfile,start=os.getcwd())
if snake_p.returncode!=0:
warn(f"sk: Complete log: {rellog}")
return snake_p.returncode
import sys
import glob
###########################################
# Workflow variables
###########################################
# import the scikick.yml config file
skconfig = ScikickConfig(need_pages=False) # no pages means empty site
# Getting properties
report_dir = skconfig.report_dir # directories
data_parent = "output"
# scikick system files
exe_dir = get_sk_exe_dir()
workflow_dir = os.path.join(exe_dir, "workflow")
snake_src = get_sk_snakefile()
generate_html_exe = os.path.join(workflow_dir, "site_rules",
"render_minimal.R")
# Add on experimental workflows for explicit usage only
snake_includes = glob.glob(
os.path.join(workflow_dir, "experimental_rules") + "/*.smk")
###########################################
# Converting scikick config to I/O deps
# for the snakemake rules
###########################################
# Depending on upstream exes' resulting 'md'