def run_sk_dryrun(snakefile=get_sk_snakefile(), workdir=os.getcwd()): # get snakemake --dryrun output with --reason # TODO - use python API status = subprocess.run( ["snakemake", "--snakefile", snakefile, \ "--directory", workdir, \ "--dryrun", "--reason"], \ stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if status.returncode != 0: warn("sk: There was an error in snakemake --dry-run, see below") reterr(status.stdout.decode("utf-8")) stdout = status.stdout.decode("utf-8").split("\n") # Filter output for job/reason outputs only stdout = list( filter( lambda l: re.match(job_pattern, l) is not None or re.match( reason_pattern, l) is not None, stdout)) # Prevent status from return is something goes wrong #assert status.returncode == 0, "snakemake dryrun failed" # get "Job..." and "Reason:..." line pairs jobs = list(map(lambda i: stdout[i * 2], range(0, int(len(stdout) / 2)))) reasons = list( map(lambda i: stdout[i * 2 + 1], range(0, int(len(stdout) / 2)))) return jobs, reasons
def inferred_inputs(self): """Return dictionary of {exe -> [dependencies], ...} For use while defining snakemake rules I/O Assumptions: 1. All analysis keys have a rule for exe.* => exe.md 2. deps that are also exe files actually depend on exe.md 3. deps that are not exe depend on the file itself """ deps = {} for exe in self.exes: out_base = self.get_info(exe, "out_base") deps[out_base] = [exe] # script itself is an input file if isinstance(self.analysis[exe], list): for dep in self.analysis[exe]: depext = os.path.splitext(dep)[-1] depisexeable = depext.lower() in [ x.lower() for x in supported_extensions ] depisexe = dep in self.exes if depisexeable and depisexe: deps[out_base].append(self.get_info(dep, "md")) elif not depisexeable and depisexe: warn("sk: Unsupported executable found in scikick.yml") else: deps[out_base].append(dep) if self.index_exe not in self.exes: deps['index'] = [self.index_exe] return deps
def detect_snakemake_progress(line, quiet=False): ntbd_match = re.match(".*Nothing to be done..*", line) job_match = re.match("^Job .*: (.*)$",line) if ntbd_match: warn("sk: Nothing to be done") elif job_match: # sanitize system index.Rmd path job_msg = job_match.groups()[0].replace(get_sk_exe_dir(),"system's ") if job_msg[0] ==' ' and quiet: return warn("sk: " + job_msg)
def write_snakefile_arg(arg, val): """Set the specified arg's value in scikick.yml args -- name of the argument val -- value of the argument """ yml = yaml_in() if "snakefile_args" not in yml.keys(): yml["snakefile_args"] = dict() yml["snakefile_args"][arg] = val warn(f"sk: Argument {arg} set to {yml['snakefile_args'][arg]}") yaml_dump(yml)
def yaml_check(config): """Checks to be run Check for unsupported fields in scikick.yml Check for project version vs scikick install version config -- dict of scikick.yml """ for k in config.keys(): if k not in supported_yaml_fields: warn(f"sk: Warning: Unrecognized scikick.yml field '{k}'") ## Checks that will be fixed and written back to scikick.yml if needed write_fixes = False if "version_info" not in config.keys(): warn( "sk: Warning: unknown project scikick version, setting to current") config = add_version_info(config) write_fixes = True else: if not config["version_info"]["scikick"] == scikick.__version__: warn( "sk: Warning: This project was not built on this version of scikick" ) if write_fixes: warn("sk: Writing fixes to scikick.yml") yaml_dump(config)
def init_dirs(): """Create default project directories""" project_dir = os.getcwd() dirs_to_create = ['report', 'input', 'output', 'code'] made_dirs=[] for curr_dir in dirs_to_create: inproj_dir = os.path.join(project_dir, curr_dir) if not os.path.exists(inproj_dir): os.mkdir(inproj_dir) made_dirs.append(curr_dir) anything_created = len(made_dirs) > 0 if anything_created: warn("sk: Created dir(s): %s" % ', '.join(made_dirs)) else: warn("sk: All default directories already exist")
def detect_snakemake_error(line): """ Return 1 if a snakemake error was detected """ # Common Snakemake errors: snakemake_lock_match = re.match("Error: Directory cannot be locked.*", line) missing_input_match = re.match("Missing input files for rule .*", line) wflow_error_match = re.match("WorkflowError:.*", line) # Snakemake errors (errors not related to analysis code) ret=1 if snakemake_lock_match: warn("sk: Error: Directory cannot be locked") warn("sk: Error: A snakemake process might already be running. To unlock, run:") warn("sk: Error: sk run -v -s --unlock") elif missing_input_match: warn("sk: Error: Missing files. Run 'sk status' to see which") elif wflow_error_match: warn("sk: Error: Snakemake WorkflowError") else: ret=0 return ret
def sk_run(args): """Run the workflow""" # check for empty analysis unless a script will be added need_pages = args.script is None skconfig = ScikickConfig(need_pages=need_pages) reportdir = skconfig.report_dir if args.snakeargs is not None: snakeargs = " ".join(args.snakeargs) warn(f"sk: Snakemake arguments received: {snakeargs}") else: snakeargs = None retcode = run_snakemake(snakefile=get_sk_snakefile(), \ workdir=os.getcwd(), \ dryrun=args.dryrun, \ snakeargs=snakeargs, \ verbose=args.verbose, \ rmds=args.script, \ quiet=args.quiet) sys.exit(retcode)
def sk_mv(args): """Rename an Rmd in scikick.yml and associated files""" config = yaml_in(need_pages=True) # multiple args src = [os.path.normpath(p) for p in args.src] # only a single arg dest = [os.path.normpath(p) for p in args.dest] #check if src and dest are valid sk_move_check(src, dest) # save the (src, dest) pairs (to be used with moves in scikick.yml) (new_src, new_dest) = sk_move_prepare_src_dest(src, dest) new_src = list(map(os.path.normpath, new_src)) new_dest = list(map(os.path.normpath, new_dest)) # leave only unique pairs of (src, dest) files mv_dict = ordereddict() for k in new_src: i = new_src.index(k) mv_dict[new_src[i]] = new_dest[i] # perform the move operation (using the initial args) for s in src: git_retcode = 0 if args.git: print(f"sk: git mv {s} {dest[0]}") git_res = subprocess.run(f"git mv {s} {dest[0]}", shell=True, \ stderr=subprocess.PIPE) git_retcode = git_res.returncode if git_retcode != 0: warn("sk: Warning: Git returned an error:") warn(git_res.stderr.decode().strip()) warn("sk: Warning: Falling back to mv") if (git_retcode != 0) or (not args.git): print(f"sk: mv {s} {dest[0]}") shutil.move(s, dest[0]) sk_move_extras(mv_dict)
def sk_move_check(src, dest): """Perform checks on src and dest and quit if bad arguments are given src -- list of files to move des -- list containing the file/dir to move to """ config = yaml_in() for s in src: if not os.path.exists(s): reterr(f"sk: Error: file or directory {s} doesn't exist") if len(src) > 1: if not os.path.isdir(dest[0]): reterr("sk: Error: moving multiple files to a single one") elif len(src) == 1 and (not os.path.isdir(dest[0])): old_ext = os.path.splitext(src[0])[1] new_ext = os.path.splitext(dest[0])[1] if old_ext.lower() != new_ext.lower(): warn("sk: Warning: changing file extension") if (src[0] in config["analysis"].keys()) and \ (new_ext.lower() not in map(str.lower, supported_extensions)): reterr( f"sk: Error: only extensions {', '.join(supported_extensions)} are supported ({new_ext} given)" )
def add_dep_checks(fname, ymli, force, dep): """ Check if dependency can be added to fname """ if not os.path.isfile(dep): warn(f"sk: Warning: {dep} does not exist or is not a file") fdeps = ymli['analysis'][fname] if fdeps is not None: if dep in fdeps: warn(f"sk: {dep} is already a dependency of {fname}") return False if True in [i in dep for i in wildcard_symbols]: warn( f"sk: Error: Filename ({dep}) cannot have wildcard symbols ({' '.join(wildcard_symbols)})" ) return False return True
def init_yaml(): """Create an initial scikick.yml config file and update the project scikick version""" template_yaml_path = os.path.join(get_sk_exe_dir(), 'usr/scikick.yml') project_dir = os.getcwd() proj_yaml_path = os.path.join(project_dir, "scikick.yml") yml_loader = ruamel.yaml.YAML() check_requirements() if os.path.exists(proj_yaml_path): warn("sk: File scikick.yml already exists, adding current scikick version") yml_out = yml_loader.load(open(proj_yaml_path, "r")) else: warn("sk: Importing template analysis configuration file") yml_out = yml_loader.load(open(template_yaml_path, "r")) yml_out = add_version_info(yml_out) warn("sk: Writing to scikick.yml") yml_loader.dump(yml_out, open(proj_yaml_path,"w"))
def yaml_in(ymlpath='scikick.yml', need_pages=False): """Read scikick.yml. Returns an ordereddict. need_pages -- logical, whether to error if analysis is empty """ #Exit with an error message if scikick.yml is not found if not os.path.isfile(ymlpath): reterr(f"sk: Error: {ymlpath} not found," + \ "to get a template, run\nsk: sk init") ymli = yaml.YAML() ymli = ymli.load(open(ymlpath, "r")) if ymli is None: warn("sk: Warning: scikick.yml is empty") ymli = dict() ## Checks that will be modified for this read-in only # make sure that mandatory fields are present if "analysis" not in ymli.keys(): if need_pages: reterr("sk: Error: no pages have been added to scikick.yml, " + \ "this can be done with\nsk: sk add my.rmd") else: warn("sk: Warning: scikick.yml is missing analysis field") ymli["analysis"] = ordereddict() else: if ymli["analysis"] is None: ymli["analysis"] = ordereddict() if len(ymli["analysis"]) == 0: if need_pages: reterr("sk: Error: no pages have been added to scikick.yml, " + \ "this can be done with\nsk: sk add my.rmd") else: ymli["analysis"] = ordereddict() if "reportdir" not in ymli.keys(): warn("sk: Warning: scikick.yml is missing reportdir field") ymli["reportdir"] = "" return ymli
def init_git(): """Add certain entries to .gitignore""" usr_dir = os.path.join(get_sk_exe_dir(), 'usr') gitignore = ".gitignore" anything_appended = False if copy_file(os.path.join(usr_dir, gitignore), \ os.path.join(gitignore)): warn("sk: File .gitignore created") else: existing_gitignore = open(gitignore, 'r').readlines() template_gitignore = open(os.path.join(usr_dir, \ gitignore), 'r').readlines() append_existing = open(gitignore, "a") for ignore_entry in template_gitignore: if ignore_entry.strip() not in map(str.strip, \ existing_gitignore): append_existing.writelines(ignore_entry) warn("sk: Added \'%s\' to .gitignore" % \ ignore_entry.rstrip()) anything_appended = True append_existing.close() if not anything_appended: warn("sk: .gitignore already has all the required entries")
def add(files, deps=list(), force=False, copy_deps=None): """ Add files and dependencies to them files -- page file list deps -- dependency file list force -- bool whether to add additional index files copy_deps -- file to copy dependencies from """ if deps is None: deps = list() ymli = yaml_in() if copy_deps is not None: # copy_deps(src,dest) copy_deps = copy_deps[0] if copy_deps not in ymli["analysis"]: reterr(f"sk: Error: file {copy_deps} is not included") deps2copy = ymli["analysis"][copy_deps] if not ((deps2copy is None) or (len(deps2copy) == 0)): deps = list(set(deps + deps2copy)) warn(f"sk: Copying {copy_deps} dependencies") else: warn(f"sk: Warning: {copy_deps} has no dependencies") # add files for fname in files: # Add script # Should the script be added? add_fname = add_check(fname, ymli, force, deps) if add_fname: # add near scripts in the same directory if len(ymli["analysis"].keys()) > 0: commpath = os.path.commonpath(list(ymli["analysis"].keys())) else: commpath = "" tab_name = os.path.dirname(rm_commdir(fname, commpath)) all_tabs = list( map(lambda f: os.path.dirname(rm_commdir(f, commpath)), ymli["analysis"].keys())) tab_matches = list( filter(lambda i: all_tabs[i] == tab_name, range(len(all_tabs)))) if (tab_name != "") and (len(tab_matches) != 0): ymli["analysis"].insert(tab_matches[-1] + 1, fname, []) else: ymli['analysis'][fname] = None warn(f"sk: Added {fname}") # Add dependencies for dep in deps: # Should the dep be added? add_dep = add_dep_checks(fname, ymli, force, dep) if add_dep: if ymli['analysis'][fname] is None: ymli['analysis'][fname] = [] ymli['analysis'][fname].append(dep) warn(f"sk: Added dependency {dep} to {fname}") if dep in ymli["analysis"].keys(): warn( f"sk: {fname} will be executed after any executions of {dep}" ) else: warn( f"sk: {fname} will be executed after any modifications to {dep}" ) yaml_dump(ymli)
def sk_move_extras(mv_dict): """Performs move operations specific to scikick Moves md files, figures in output/ directories and renames files in scikick.yml mv_dict -- dict with keys as src and values as dest files """ # Moving mds, knitmetas and output figures in out_md/; ## No need to change _site.ymls, since ## they are recreated after each change in scikick.yml yaml_dict = yaml_in() analysis = yaml_dict["analysis"] reportdir = yaml_dict["reportdir"] for src, dest in mv_dict.items(): # if not in analysis.keys -> regular file # in this case, rename and continue if src not in analysis.keys(): if 1 == rename(src, dest): warn("sk: %s renamed to %s in ./scikick.yml" % (src, dest)) else: warn("sk: Warning: %s not found in ./scikick.yml" % src) continue md_rootdir = os.path.join(reportdir, "out_md") md_destdir = os.path.join(md_rootdir, os.path.dirname(dest)) md_srcdir = os.path.join(md_rootdir, os.path.dirname(src)) if not os.path.isdir(md_destdir): os.makedirs(md_destdir) # Move .md md_src = os.path.join(md_rootdir, os.path.splitext(src)[0] + ".md") md_dest = os.path.join( md_destdir, os.path.splitext(os.path.basename(dest))[0] + ".md") if os.path.isfile(md_src): shutil.move(md_src, md_dest) # Move knitmeta k_src = sub(pattern="\.md$", repl=".knitmeta.RDS", string=md_src) k_dest = sub(pattern="\.md$", repl=".knitmeta.RDS", string=md_dest) if os.path.isfile(k_src): shutil.move(k_src, k_dest) # Move markdown outputs (figures) tabname_src = sub(string=os.path.basename(md_src), pattern="\.md$", repl="") tabname_dest = sub(string=os.path.basename(md_dest), pattern="\.md$", repl="") # "figure" must match execute_code.R fig.path md_srcfigdir = os.path.join(md_srcdir, "figure", tabname_src) md_destfigdir = os.path.join(md_destdir, "figure", tabname_dest) if os.path.isdir(md_srcfigdir): dest_dirname = os.path.dirname(md_srcfigdir) if not os.path.isdir(dest_dirname): os.makedirs(dest_dirname, exist_oke=True) print(f"sk: mv {md_srcfigdir} {md_destfigdir}") shutil.move(md_srcfigdir, md_destfigdir) # rename the figure/ directory name in the dest md ## get the initial timestamp initial_timestamp = os.path.getmtime(md_dest) md_file = open(md_dest, 'r+') md_lines = [ sub(string=line, pattern=f'src="figure/{tabname_src}', repl=f'src="figure/{tabname_dest}') for line in md_file ] md_file.seek(0) for l in md_lines: md_file.write(l) md_file.close() # set the initial timestamp back to avoid reexecution os.utime(md_dest, (initial_timestamp, initial_timestamp)) # rename all entries in scikick.yml from from to f_dest if 1 == rename(src, dest): warn("sk: %s renamed to %s in ./scikick.yml" % (src, dest)) else: warn("sk: Warning: %s not found in ./scikick.yml" % src)
def sk_init(args): """Initialize scikick project""" if not (args.git or args.dirs or args.yaml or args.readme or args.demo): args.yaml = True warn("sk: No arguments supplied, defaulting to sk init -y") init(args) warn("sk: See the below arguments for other " + \ "possible sk init components") parser_init.print_help() warn("sk: To add an R/Rmd script to the analysis use") warn("sk: sk add my.rmd") warn("sk: Then, to execute the workflow") warn("sk: sk run") else: init(args)
def snake_status( snakefile=get_sk_snakefile(), workdir=os.getcwd(), verbose=False, rmd=None): """Print workflow status snakefile -- string (path to the main snakefile) workdir -- string verbose -- bool rmd -- string (show status for just this file) """ skconf = ScikickConfig( ) # OR status with no pages just indicates whether index.html exists jobs, reasons = run_sk_dryrun(snakefile, workdir) # split jobs/reasons into types def subset_jobs(jobs, reasons, jtype): # Trying to use only exe_job info ret_jobs = [] ret_reasons = [] for i, _ in enumerate(jobs): if job_type(jobs[i]) == jtype: ret_jobs.append(jobs[i]) ret_reasons.append(reasons[i]) return ret_jobs, ret_reasons exe_jobs, exe_reasons = subset_jobs(jobs, reasons, "exe_to_md") md_jobs, md_reasons = subset_jobs(jobs, reasons, "md_to_html") unknown_jobs, unknown_reasons = subset_jobs(jobs, reasons, "unknown") exes = skconf.exes if skconf.index_exe not in exes: exes.append(skconf.index_exe) ## Parsing job/reason info from snakemake outputs # each function matches a snakemake reason string # get expected input updates for each script expected_input_updates = get_expected_input_updates( exes, exe_reasons, exe_jobs) # get updated inputs for each script updated_inputs = get_updated_inputs(exes, exe_reasons, exe_jobs) # get missing outputs (which are to be generated) missing_outs = missing_outputs(exe_reasons) ## Parsing job info only # get which scripts will be executed (processed as exe => md) exec_scripts = [ re.match(exe_pattern, exe_job).groups()[0] for exe_job in exe_jobs ] # get status markers (codes ---) for each script markers = file_markers(skconf, updated_inputs, expected_input_updates, missing_outs, exec_scripts) # Add site status for job in md_jobs: match = re.match(htmlgen_pattern, job) # if the script has an html step fill in the gaps with --- if match is not None: out_base = match.groups()[0] md_job_exe = skconf.get_info(out_base, "exe") markers[md_job_exe] = [ "-" if x == " " else x for x in markers[md_job_exe] ] # Checking for valid codes # Probably better to be more careful than do this nonexe_chars = [' ', '-'] for exe in exec_scripts: if all([x in nonexe_chars for x in markers[exe]]): markers[exe] = ['*', '-', '-'] warn( f"sk: Warning: {exe} must execute for unknown reasons, (was scikick updated since last run?)" ) # Print status or a subset if rmd is not None: # print only the status of rmd and dependent files reduced_analysis = {k: skconf.analysis[k] for k in \ filter(lambda k: k in skconf.exes, \ flatten_dependency_tree(rmd, skconf))} print_status(reduced_analysis, markers, verbose) else: config = skconf.analysis # Adjustment for system index if skconf.index_exe not in config.keys(): config['system index (homepage)'] = list() # has no inputs markers['system index (homepage)'] = markers[skconf.index_exe] print_status(config, markers, verbose)
def run_snakemake(snakefile=get_sk_snakefile(), workdir=os.getcwd(), \ verbose=False, dryrun=False, snakeargs=None, rmds=[], quiet=False): """Run snakemake with specified arguments snakefile -- string (path to the main snakefile) workdir -- string verbose -- bool dryrun -- bool snakeargs -- list (list of additional arguments to snakemake) rmds -- string rmd who's output should be targetted """ exe_dir = get_sk_exe_dir() loghandler = os.path.join(exe_dir, 'workflow/loghandler.py') skconf = ScikickConfig() yml = skconf.config # logfile created by snakemake snake_logfile="" ### Construct call to snakemake # before 'snakemake' env_vars = f'SINGULARITY_BINDPATH="{exe_dir}"' # after 'snakemake' snakemake_args = "" snakemake_args += f" --snakefile {snakefile}" snakemake_args += f" --directory '{workdir}'" snakemake_args += " --cores 1" snakemake_args += f" --log-handler-script {loghandler}" # Translate Rmd script to HTML target # TODO - factor out # TODO - move this to sk_run as an additional snake_arg # to reduce skconfig read ins target_arg = "" if len(rmds) > 0: for rmd in rmds: # Try to add rmd if it is not in scikick.yml if yml['analysis'] is None: rmd_found = False elif rmd not in yml["analysis"].keys(): rmd_found = False else: rmd_found = True if not rmd_found: warn(f"sk: Warning: {rmd} was not found in scikick.yml") if os.path.exists(rmd): scikick.yaml.add([rmd]) # Must reload or modify the yml since it changed skconf = ScikickConfig() yml = skconf.config else: warn(f"sk: Warning: Will not try to add non-existing file {rmd}") return 0 # Set target. Index file is treated differently index_rmds = get_indexes(yml) if (len(index_rmds) == 1) and (index_rmds[0] == rmd): target_arg += " " + os.path.join(yml["reportdir"], \ "out_html", "index.html") else: target_arg += " " + os.path.join(yml["reportdir"], \ "out_html", os.path.splitext(rmd)[0] + ".html") # set more snakemake arguments if dryrun: snakemake_args += " --dry-run" # Add user defined snakemake arguments if snakeargs is not None: snakemake_args += f" {snakeargs}" if 'snakemake_args' in yml.keys() and yml['snakemake_args'] is not None: warn("sk: Warning: snakemake_args is deprecated") snakemake_args += f" {' '.join(yml['snakemake_args'])}" # add the implied targets (html of Rmd) snakemake_args += f" {target_arg}" # Check for a user defined snakefile (imported by scikick Snakefile) user_snakefile = os.path.join(os.getcwd(), "Snakefile") if os.path.isfile(user_snakefile): warn("sk: Including Snakefile found in project directory") ### Execution if verbose: warn("sk: Starting snakemake") cmd = f"{env_vars} snakemake {snakemake_args}" print(cmd) sys.exit(subprocess.call(cmd, shell=True)) else: snake_p = subprocess.Popen(f"snakemake {snakemake_args}", \ shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logs=[] sm_err = 0 page_err = 0 while True: line = snake_p.stdout.readline().decode('utf-8') if not line: break # Capture logs logs.append(line) # Report progress detect_snakemake_progress(line,quiet) page_err = detect_page_error(line) or page_err # In case of snakemake error start writing stderr sm_err += detect_snakemake_error(line) if sm_err: sys.stderr.write(line) logfile_name_match = re.match("^SK INTERNAL: logfile (.*)$", line) if logfile_name_match is not None: # Use the first found match if snake_logfile == "": snake_logfile = logfile_name_match.groups()[0] snake_p.wait() if snake_p.returncode != 0: if not page_err: warn("sk: Error: Snakemake returned a non-zero return code") if not sm_err: warn("sk: Warning: scikick was unable to find snakemake error in logs, dumping stderr...") for line in logs: sys.stderr.write(line) return snake_p.returncode else: if not os.path.exists(skconf.homepage): warn(f"sk: Warning: Expected homepage {skconf.homepage} is missing") if snake_logfile != "": rellog=os.path.relpath(snake_logfile,start=os.getcwd()) if snake_p.returncode!=0: warn(f"sk: Complete log: {rellog}") return snake_p.returncode
def rm(files, deps): """ Delete files and dependencies from them files - page file list deps - dependency file list """ if deps is None: deps = list() ymli = yaml_in(need_pages=True) for fname in files: # check if rmd included if fname not in ymli['analysis'].keys(): warn(f"sk: Warning: File {fname} not included") continue # delete script entry if no dependencies specified if len(deps) == 0: del ymli['analysis'][fname] warn(f"sk: {fname} removed") # Check if fname was a dependency for other scripts for val in ymli['analysis'].values(): if val is not None: if fname in val: warn( f"sk: Warning: {fname} is still a dependency of other scripts" ) warn( f"sk: Use sk del -d {fname} <script> to change this" ) # delete only deps if deps specified else: if ymli['analysis'][fname] is None: warn(f"sk: Warning: File {fname} has no dependencies") continue for dep in deps: if dep in ymli['analysis'][fname]: ymli['analysis'][fname].remove(dep) warn(f"sk: dependency {dep} removed from {fname}") else: warn(f"sk: no dependency {dep} found for {fname}") if len((ymli['analysis'][fname])) == 0: ymli['analysis'][fname] = None if os.path.splitext(os.path.basename(fname))[0] == "index": index_list = get_indexes(ymli) if len(index_list) == 0: warn(f"sk: Using system template index.Rmd as homepage") os.utime( os.path.join(get_sk_exe_dir(), "workflow", "notebook_rules", "index.Rmd"), None) elif len(index_list) == 1: os.utime(index_list[0], None) yaml_dump(ymli)
def add_check(fname, ymli, force, deps): """Performs a check if fname can be added to scikick.yml as a key""" pages = ymli["analysis"].keys() if fname in pages: warn(f"sk: Found existing entry for {fname}") return False # filenames cannot currently have wildcard symbols if True in [i in fname for i in wildcard_symbols]: warn( f"sk: Error: Filename ({fname}) cannot have wildcard symbols ({' '.join(wildcard_symbols)})" ) return False # check if the file extension is supported fext = os.path.splitext(fname)[-1] if fext.lower() == ".ipynb": warn( "sk: Warning: .ipynb use in Scikick is experimental and requires installation of jupyter" ) f_exe_support = fext.lower() in [x.lower() for x in supported_extensions] if not f_exe_support: extension_list_str = ', '.join(supported_extensions) warn("sk: Error: Only %s files can be added as pages (%s)" % \ (extension_list_str, fname)) return False # error if the directory doesn't exist dirname = os.path.dirname(fname) if dirname != "": if not os.path.isdir(dirname): reterr(f"sk: Error: Directory {dirname} does not exist.") # create the file if it doesn't exist if not os.path.isfile(fname): warn(f"sk: Warning: File {fname} doesn't exist") warn(f"sk: Creating new file {fname}") open(fname, "a").close() if fname not in pages: # Check for other files with same basename (and therefore same md output file) fname_base = os.path.splitext(fname)[0] page_basenames = map(lambda x: os.path.splitext(x)[0], pages) page_basename_exists = fname_base in page_basenames if page_basename_exists: warn( f"sk: Error: Page {fname_base} is already to be compiled from another file." ) return False # check for "index.Rmd"s index_list = get_indexes(ymli) if os.path.splitext(os.path.basename(fname))[0] == "index": if len(index_list) == 0: warn( f"sk: An index file {fname} has been added and will be used as the homepage" ) # touch the added index file to ensure execution os.utime(fname, None) elif len(index_list) == 1: if not force: reterr(f"sk: Error: An index file {index_list[0]} already exists\n" + \ "sk: Error: An additional one can be added, but neither will be used as a homepage\n" + \ f"sk: Error: To persist, use 'sk add --force {fname}'") else: warn( f"sk: Warning: A redundant index file has been added\n" + "sk: Warning: Neither of the added index files will be used as a homepage" ) os.utime( os.path.join(get_sk_exe_dir(), "workflow", "notebook_rules", "index.Rmd"), None) elif len(index_list) > 1: warn( f"sk: Warning: A redundant index file has been added\n" + "sk: Warning: Neither of the added index files will be used as a homepage" ) return True