def index_exe(self): """ Which source file to use for index page """ if len(self.index_exes) == 1: return self.index_exes[0] else: return os.path.join(get_sk_exe_dir(), "workflow", "notebook_rules", 'index.Rmd')
def detect_snakemake_progress(line, quiet=False): ntbd_match = re.match(".*Nothing to be done..*", line) job_match = re.match("^Job .*: (.*)$",line) if ntbd_match: warn("sk: Nothing to be done") elif job_match: # sanitize system index.Rmd path job_msg = job_match.groups()[0].replace(get_sk_exe_dir(),"system's ") if job_msg[0] ==' ' and quiet: return warn("sk: " + job_msg)
def main(): yaml = YAML(typ="rt") skconfig = ScikickConfig() # get tab strucutre tabs = get_tabs(skconfig) site_yaml_files = skconfig.get_site_yaml_files() giturl = git_repo_url() if giturl != '.': # get git repo url nav_more = {"text": "More", \ "menu": [{"text" : "Git Repository", "href" : giturl}]} else: nav_more = {} # translating the desired layout (get_tabs) to _site.yml format for site_yaml_file in site_yaml_files: nav_left = list() # path from the site yaml to the output root path_to_root = relpath(join(skconfig.report_dir,'out_md'),start=dirname(site_yaml_file)) for tab, items in tabs.items(): human_text = clean_name(tab) # if first value is the key, this is a file tabisfile = basename(items[0]) == tab if tabisfile: path_from_site_to_html = join(path_to_root,items[0]) this_item = {"text": human_text, "href": "%s.html" % path_from_site_to_html} else: this_item = {"text": human_text, "menu":[]} for item in items: path_from_site_to_html = join(path_to_root,item) sub_item = {"text": clean_name(basename(item)), "href": "%s.html" % path_from_site_to_html} this_item['menu'].append(sub_item) nav_left.append(this_item) site_yaml= { "navbar": {"title": clean_name(basename(getcwd())), \ "left": nav_left}} if nav_more != {}: site_yaml["navbar"]["right"] = [nav_more] if 'output' in skconfig.config.keys(): output_yaml = skconfig.config else: # TODO merge with scikick.yml output_yaml = yaml.load(open(join(get_sk_exe_dir(),"workflow/site_rules/default_output.yml"),"r")) output_yaml['output']['rmarkdown::html_document']['pandoc_args'] = '--resource-path=.:' + path_to_root + '/../../' site_yaml.update(output_yaml) yaml.indent(sequence=4, mapping=4, offset=0) yaml.dump(site_yaml, open(site_yaml_file, "w"))
def rm(files, deps): """ Delete files and dependencies from them files - page file list deps - dependency file list """ if deps is None: deps = list() ymli = yaml_in(need_pages=True) for fname in files: # check if rmd included if fname not in ymli['analysis'].keys(): warn(f"sk: Warning: File {fname} not included") continue # delete script entry if no dependencies specified if len(deps) == 0: del ymli['analysis'][fname] warn(f"sk: {fname} removed") # Check if fname was a dependency for other scripts for val in ymli['analysis'].values(): if val is not None: if fname in val: warn( f"sk: Warning: {fname} is still a dependency of other scripts" ) warn( f"sk: Use sk del -d {fname} <script> to change this" ) # delete only deps if deps specified else: if ymli['analysis'][fname] is None: warn(f"sk: Warning: File {fname} has no dependencies") continue for dep in deps: if dep in ymli['analysis'][fname]: ymli['analysis'][fname].remove(dep) warn(f"sk: dependency {dep} removed from {fname}") else: warn(f"sk: no dependency {dep} found for {fname}") if len((ymli['analysis'][fname])) == 0: ymli['analysis'][fname] = None if os.path.splitext(os.path.basename(fname))[0] == "index": index_list = get_indexes(ymli) if len(index_list) == 0: warn(f"sk: Using system template index.Rmd as homepage") os.utime( os.path.join(get_sk_exe_dir(), "workflow", "notebook_rules", "index.Rmd"), None) elif len(index_list) == 1: os.utime(index_list[0], None) yaml_dump(ymli)
def init_yaml(): """Create an initial scikick.yml config file and update the project scikick version""" template_yaml_path = os.path.join(get_sk_exe_dir(), 'usr/scikick.yml') project_dir = os.getcwd() proj_yaml_path = os.path.join(project_dir, "scikick.yml") yml_loader = ruamel.yaml.YAML() check_requirements() if os.path.exists(proj_yaml_path): warn("sk: File scikick.yml already exists, adding current scikick version") yml_out = yml_loader.load(open(proj_yaml_path, "r")) else: warn("sk: Importing template analysis configuration file") yml_out = yml_loader.load(open(template_yaml_path, "r")) yml_out = add_version_info(yml_out) warn("sk: Writing to scikick.yml") yml_loader.dump(yml_out, open(proj_yaml_path,"w"))
def run_demo_stage1(): print_demo("A demo project will be used to demonstrate some features of scikick.") print("") print_demo("---------- Starting a New Project ----------") print_demo("A new scikick project can be initialized with sk init which will:") print_demo("check some software dependencies,") print_demo("add the scikick.yml config file (-y),") print_demo("and make some useful directories (-d).") print_exec("sk init -yd") # copy the files demo_templatedir = os.path.join(get_sk_exe_dir(), "demo") for f in ["generate.Rmd", "PCA.Rmd", "analysis_config.txt", \ "PC_score_statistics.Rmd", "index.Rmd"]: shutil.copy(os.path.join(demo_templatedir, f), os.path.join("code", f)) print_demo("Demo project has been initialized.") print_demo("Run sk init --demo again to continue.")
def init_git(): """Add certain entries to .gitignore""" usr_dir = os.path.join(get_sk_exe_dir(), 'usr') gitignore = ".gitignore" anything_appended = False if copy_file(os.path.join(usr_dir, gitignore), \ os.path.join(gitignore)): warn("sk: File .gitignore created") else: existing_gitignore = open(gitignore, 'r').readlines() template_gitignore = open(os.path.join(usr_dir, \ gitignore), 'r').readlines() append_existing = open(gitignore, "a") for ignore_entry in template_gitignore: if ignore_entry.strip() not in map(str.strip, \ existing_gitignore): append_existing.writelines(ignore_entry) warn("sk: Added \'%s\' to .gitignore" % \ ignore_entry.rstrip()) anything_appended = True append_existing.close() if not anything_appended: warn("sk: .gitignore already has all the required entries")
def add_check(fname, ymli, force, deps): """Performs a check if fname can be added to scikick.yml as a key""" pages = ymli["analysis"].keys() if fname in pages: warn(f"sk: Found existing entry for {fname}") return False # filenames cannot currently have wildcard symbols if True in [i in fname for i in wildcard_symbols]: warn( f"sk: Error: Filename ({fname}) cannot have wildcard symbols ({' '.join(wildcard_symbols)})" ) return False # check if the file extension is supported fext = os.path.splitext(fname)[-1] if fext.lower() == ".ipynb": warn( "sk: Warning: .ipynb use in Scikick is experimental and requires installation of jupyter" ) f_exe_support = fext.lower() in [x.lower() for x in supported_extensions] if not f_exe_support: extension_list_str = ', '.join(supported_extensions) warn("sk: Error: Only %s files can be added as pages (%s)" % \ (extension_list_str, fname)) return False # error if the directory doesn't exist dirname = os.path.dirname(fname) if dirname != "": if not os.path.isdir(dirname): reterr(f"sk: Error: Directory {dirname} does not exist.") # create the file if it doesn't exist if not os.path.isfile(fname): warn(f"sk: Warning: File {fname} doesn't exist") warn(f"sk: Creating new file {fname}") open(fname, "a").close() if fname not in pages: # Check for other files with same basename (and therefore same md output file) fname_base = os.path.splitext(fname)[0] page_basenames = map(lambda x: os.path.splitext(x)[0], pages) page_basename_exists = fname_base in page_basenames if page_basename_exists: warn( f"sk: Error: Page {fname_base} is already to be compiled from another file." ) return False # check for "index.Rmd"s index_list = get_indexes(ymli) if os.path.splitext(os.path.basename(fname))[0] == "index": if len(index_list) == 0: warn( f"sk: An index file {fname} has been added and will be used as the homepage" ) # touch the added index file to ensure execution os.utime(fname, None) elif len(index_list) == 1: if not force: reterr(f"sk: Error: An index file {index_list[0]} already exists\n" + \ "sk: Error: An additional one can be added, but neither will be used as a homepage\n" + \ f"sk: Error: To persist, use 'sk add --force {fname}'") else: warn( f"sk: Warning: A redundant index file has been added\n" + "sk: Warning: Neither of the added index files will be used as a homepage" ) os.utime( os.path.join(get_sk_exe_dir(), "workflow", "notebook_rules", "index.Rmd"), None) elif len(index_list) > 1: warn( f"sk: Warning: A redundant index file has been added\n" + "sk: Warning: Neither of the added index files will be used as a homepage" ) return True
def run_snakemake(snakefile=get_sk_snakefile(), workdir=os.getcwd(), \ verbose=False, dryrun=False, snakeargs=None, rmds=[], quiet=False): """Run snakemake with specified arguments snakefile -- string (path to the main snakefile) workdir -- string verbose -- bool dryrun -- bool snakeargs -- list (list of additional arguments to snakemake) rmds -- string rmd who's output should be targetted """ exe_dir = get_sk_exe_dir() loghandler = os.path.join(exe_dir, 'workflow/loghandler.py') skconf = ScikickConfig() yml = skconf.config # logfile created by snakemake snake_logfile="" ### Construct call to snakemake # before 'snakemake' env_vars = f'SINGULARITY_BINDPATH="{exe_dir}"' # after 'snakemake' snakemake_args = "" snakemake_args += f" --snakefile {snakefile}" snakemake_args += f" --directory '{workdir}'" snakemake_args += " --cores 1" snakemake_args += f" --log-handler-script {loghandler}" # Translate Rmd script to HTML target # TODO - factor out # TODO - move this to sk_run as an additional snake_arg # to reduce skconfig read ins target_arg = "" if len(rmds) > 0: for rmd in rmds: # Try to add rmd if it is not in scikick.yml if yml['analysis'] is None: rmd_found = False elif rmd not in yml["analysis"].keys(): rmd_found = False else: rmd_found = True if not rmd_found: warn(f"sk: Warning: {rmd} was not found in scikick.yml") if os.path.exists(rmd): scikick.yaml.add([rmd]) # Must reload or modify the yml since it changed skconf = ScikickConfig() yml = skconf.config else: warn(f"sk: Warning: Will not try to add non-existing file {rmd}") return 0 # Set target. Index file is treated differently index_rmds = get_indexes(yml) if (len(index_rmds) == 1) and (index_rmds[0] == rmd): target_arg += " " + os.path.join(yml["reportdir"], \ "out_html", "index.html") else: target_arg += " " + os.path.join(yml["reportdir"], \ "out_html", os.path.splitext(rmd)[0] + ".html") # set more snakemake arguments if dryrun: snakemake_args += " --dry-run" # Add user defined snakemake arguments if snakeargs is not None: snakemake_args += f" {snakeargs}" if 'snakemake_args' in yml.keys() and yml['snakemake_args'] is not None: warn("sk: Warning: snakemake_args is deprecated") snakemake_args += f" {' '.join(yml['snakemake_args'])}" # add the implied targets (html of Rmd) snakemake_args += f" {target_arg}" # Check for a user defined snakefile (imported by scikick Snakefile) user_snakefile = os.path.join(os.getcwd(), "Snakefile") if os.path.isfile(user_snakefile): warn("sk: Including Snakefile found in project directory") ### Execution if verbose: warn("sk: Starting snakemake") cmd = f"{env_vars} snakemake {snakemake_args}" print(cmd) sys.exit(subprocess.call(cmd, shell=True)) else: snake_p = subprocess.Popen(f"snakemake {snakemake_args}", \ shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logs=[] sm_err = 0 page_err = 0 while True: line = snake_p.stdout.readline().decode('utf-8') if not line: break # Capture logs logs.append(line) # Report progress detect_snakemake_progress(line,quiet) page_err = detect_page_error(line) or page_err # In case of snakemake error start writing stderr sm_err += detect_snakemake_error(line) if sm_err: sys.stderr.write(line) logfile_name_match = re.match("^SK INTERNAL: logfile (.*)$", line) if logfile_name_match is not None: # Use the first found match if snake_logfile == "": snake_logfile = logfile_name_match.groups()[0] snake_p.wait() if snake_p.returncode != 0: if not page_err: warn("sk: Error: Snakemake returned a non-zero return code") if not sm_err: warn("sk: Warning: scikick was unable to find snakemake error in logs, dumping stderr...") for line in logs: sys.stderr.write(line) return snake_p.returncode else: if not os.path.exists(skconf.homepage): warn(f"sk: Warning: Expected homepage {skconf.homepage} is missing") if snake_logfile != "": rellog=os.path.relpath(snake_logfile,start=os.getcwd()) if snake_p.returncode!=0: warn(f"sk: Complete log: {rellog}") return snake_p.returncode
import sys import glob ########################################### # Workflow variables ########################################### # import the scikick.yml config file skconfig = ScikickConfig(need_pages=False) # no pages means empty site # Getting properties report_dir = skconfig.report_dir # directories data_parent = "output" # scikick system files exe_dir = get_sk_exe_dir() workflow_dir = os.path.join(exe_dir, "workflow") snake_src = get_sk_snakefile() generate_html_exe = os.path.join(workflow_dir, "site_rules", "render_minimal.R") # Add on experimental workflows for explicit usage only snake_includes = glob.glob( os.path.join(workflow_dir, "experimental_rules") + "/*.smk") ########################################### # Converting scikick config to I/O deps # for the snakemake rules ########################################### # Depending on upstream exes' resulting 'md'