def run_sk_dryrun(snakefile=get_sk_snakefile(), workdir=os.getcwd()):
# get snakemake --dryrun output with --reason
# TODO - use python API
status = subprocess.run(
["snakemake", "--snakefile", snakefile, \
"--directory", workdir, \
"--dryrun", "--reason"], \
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if status.returncode != 0:
warn("sk: There was an error in snakemake --dry-run, see below")
reterr(status.stdout.decode("utf-8"))
stdout = status.stdout.decode("utf-8").split("\n")
# Filter output for job/reason outputs only
stdout = list(
filter(
lambda l: re.match(job_pattern, l) is not None or re.match(
reason_pattern, l) is not None, stdout))
# Prevent status from return is something goes wrong
#assert status.returncode == 0, "snakemake dryrun failed"
# get "Job..." and "Reason:..." line pairs
jobs = list(map(lambda i: stdout[i * 2], range(0, int(len(stdout) / 2))))
reasons = list(
map(lambda i: stdout[i * 2 + 1], range(0, int(len(stdout) / 2))))
return jobs, reasons
def inferred_inputs(self):
"""Return dictionary of {exe -> [dependencies], ...}
For use while defining snakemake rules I/O
Assumptions:
1. All analysis keys have a rule for exe.* => exe.md
2. deps that are also exe files actually depend on exe.md
3. deps that are not exe depend on the file itself
"""
deps = {}
for exe in self.exes:
out_base = self.get_info(exe, "out_base")
deps[out_base] = [exe] # script itself is an input file
if isinstance(self.analysis[exe], list):
for dep in self.analysis[exe]:
depext = os.path.splitext(dep)[-1]
depisexeable = depext.lower() in [
x.lower() for x in supported_extensions
]
depisexe = dep in self.exes
if depisexeable and depisexe:
deps[out_base].append(self.get_info(dep, "md"))
elif not depisexeable and depisexe:
warn("sk: Unsupported executable found in scikick.yml")
else:
deps[out_base].append(dep)
if self.index_exe not in self.exes:
deps['index'] = [self.index_exe]
return deps
def detect_snakemake_progress(line, quiet=False):
ntbd_match = re.match(".*Nothing to be done..*", line)
job_match = re.match("^Job .*: (.*)$",line)
if ntbd_match:
warn("sk: Nothing to be done")
elif job_match:
# sanitize system index.Rmd path
job_msg = job_match.groups()[0].replace(get_sk_exe_dir(),"system's ")
if job_msg[0] ==' ' and quiet:
return
warn("sk: " + job_msg)
def write_snakefile_arg(arg, val):
"""Set the specified arg's value in scikick.yml
args -- name of the argument
val -- value of the argument
"""
yml = yaml_in()
if "snakefile_args" not in yml.keys():
yml["snakefile_args"] = dict()
yml["snakefile_args"][arg] = val
warn(f"sk: Argument {arg} set to {yml['snakefile_args'][arg]}")
yaml_dump(yml)
def yaml_check(config):
"""Checks to be run
Check for unsupported fields in scikick.yml
Check for project version vs scikick install version
config -- dict of scikick.yml
"""
for k in config.keys():
if k not in supported_yaml_fields:
warn(f"sk: Warning: Unrecognized scikick.yml field '{k}'")
## Checks that will be fixed and written back to scikick.yml if needed
write_fixes = False
if "version_info" not in config.keys():
warn(
"sk: Warning: unknown project scikick version, setting to current")
config = add_version_info(config)
write_fixes = True
else:
if not config["version_info"]["scikick"] == scikick.__version__:
warn(
"sk: Warning: This project was not built on this version of scikick"
)
if write_fixes:
warn("sk: Writing fixes to scikick.yml")
yaml_dump(config)
def init_dirs():
"""Create default project directories"""
project_dir = os.getcwd()
dirs_to_create = ['report', 'input', 'output', 'code']
made_dirs=[]
for curr_dir in dirs_to_create:
inproj_dir = os.path.join(project_dir, curr_dir)
if not os.path.exists(inproj_dir):
os.mkdir(inproj_dir)
made_dirs.append(curr_dir)
anything_created = len(made_dirs) > 0
if anything_created:
warn("sk: Created dir(s): %s" % ', '.join(made_dirs))
else:
warn("sk: All default directories already exist")
def detect_snakemake_error(line):
"""
Return 1 if a snakemake error was detected
"""
# Common Snakemake errors:
snakemake_lock_match = re.match("Error: Directory cannot be locked.*", line)
missing_input_match = re.match("Missing input files for rule .*", line)
wflow_error_match = re.match("WorkflowError:.*", line)
# Snakemake errors (errors not related to analysis code)
ret=1
if snakemake_lock_match:
warn("sk: Error: Directory cannot be locked")
warn("sk: Error: A snakemake process might already be running. To unlock, run:")
warn("sk: Error: sk run -v -s --unlock")
elif missing_input_match:
warn("sk: Error: Missing files. Run 'sk status' to see which")
elif wflow_error_match:
warn("sk: Error: Snakemake WorkflowError")
else:
ret=0
return ret
def sk_run(args):
"""Run the workflow"""
# check for empty analysis unless a script will be added
need_pages = args.script is None
skconfig = ScikickConfig(need_pages=need_pages)
reportdir = skconfig.report_dir
if args.snakeargs is not None:
snakeargs = " ".join(args.snakeargs)
warn(f"sk: Snakemake arguments received: {snakeargs}")
else:
snakeargs = None
retcode = run_snakemake(snakefile=get_sk_snakefile(), \
workdir=os.getcwd(), \
dryrun=args.dryrun, \
snakeargs=snakeargs, \
verbose=args.verbose, \
rmds=args.script, \
quiet=args.quiet)
sys.exit(retcode)
def sk_mv(args):
"""Rename an Rmd in scikick.yml and associated files"""
config = yaml_in(need_pages=True)
# multiple args
src = [os.path.normpath(p) for p in args.src]
# only a single arg
dest = [os.path.normpath(p) for p in args.dest]
#check if src and dest are valid
sk_move_check(src, dest)
# save the (src, dest) pairs (to be used with moves in scikick.yml)
(new_src, new_dest) = sk_move_prepare_src_dest(src, dest)
new_src = list(map(os.path.normpath, new_src))
new_dest = list(map(os.path.normpath, new_dest))
# leave only unique pairs of (src, dest) files
mv_dict = ordereddict()
for k in new_src:
i = new_src.index(k)
mv_dict[new_src[i]] = new_dest[i]
# perform the move operation (using the initial args)
for s in src:
git_retcode = 0
if args.git:
print(f"sk: git mv {s} {dest[0]}")
git_res = subprocess.run(f"git mv {s} {dest[0]}", shell=True, \
stderr=subprocess.PIPE)
git_retcode = git_res.returncode
if git_retcode != 0:
warn("sk: Warning: Git returned an error:")
warn(git_res.stderr.decode().strip())
warn("sk: Warning: Falling back to mv")
if (git_retcode != 0) or (not args.git):
print(f"sk: mv {s} {dest[0]}")
shutil.move(s, dest[0])
sk_move_extras(mv_dict)
def sk_move_check(src, dest):
"""Perform checks on src and dest
and quit if bad arguments are given
src -- list of files to move
des -- list containing the file/dir to move to
"""
config = yaml_in()
for s in src:
if not os.path.exists(s):
reterr(f"sk: Error: file or directory {s} doesn't exist")
if len(src) > 1:
if not os.path.isdir(dest[0]):
reterr("sk: Error: moving multiple files to a single one")
elif len(src) == 1 and (not os.path.isdir(dest[0])):
old_ext = os.path.splitext(src[0])[1]
new_ext = os.path.splitext(dest[0])[1]
if old_ext.lower() != new_ext.lower():
warn("sk: Warning: changing file extension")
if (src[0] in config["analysis"].keys()) and \
(new_ext.lower() not in map(str.lower, supported_extensions)):
reterr(
f"sk: Error: only extensions {', '.join(supported_extensions)} are supported ({new_ext} given)"
)
def add_dep_checks(fname, ymli, force, dep):
""" Check if dependency can be added to fname """
if not os.path.isfile(dep):
warn(f"sk: Warning: {dep} does not exist or is not a file")
fdeps = ymli['analysis'][fname]
if fdeps is not None:
if dep in fdeps:
warn(f"sk: {dep} is already a dependency of {fname}")
return False
if True in [i in dep for i in wildcard_symbols]:
warn(
f"sk: Error: Filename ({dep}) cannot have wildcard symbols ({' '.join(wildcard_symbols)})"
)
return False
return True
def init_yaml():
"""Create an initial scikick.yml config file and update the project scikick
version"""
template_yaml_path = os.path.join(get_sk_exe_dir(), 'usr/scikick.yml')
project_dir = os.getcwd()
proj_yaml_path = os.path.join(project_dir, "scikick.yml")
yml_loader = ruamel.yaml.YAML()
check_requirements()
if os.path.exists(proj_yaml_path):
warn("sk: File scikick.yml already exists, adding current scikick version")
yml_out = yml_loader.load(open(proj_yaml_path, "r"))
else:
warn("sk: Importing template analysis configuration file")
yml_out = yml_loader.load(open(template_yaml_path, "r"))
yml_out = add_version_info(yml_out)
warn("sk: Writing to scikick.yml")
yml_loader.dump(yml_out, open(proj_yaml_path,"w"))
def yaml_in(ymlpath='scikick.yml', need_pages=False):
"""Read scikick.yml.
Returns an ordereddict.
need_pages -- logical, whether to error if analysis is empty
"""
#Exit with an error message if scikick.yml is not found
if not os.path.isfile(ymlpath):
reterr(f"sk: Error: {ymlpath} not found," + \
"to get a template, run\nsk: sk init")
ymli = yaml.YAML()
ymli = ymli.load(open(ymlpath, "r"))
if ymli is None:
warn("sk: Warning: scikick.yml is empty")
ymli = dict()
## Checks that will be modified for this read-in only
# make sure that mandatory fields are present
if "analysis" not in ymli.keys():
if need_pages:
reterr("sk: Error: no pages have been added to scikick.yml, " + \
"this can be done with\nsk: sk add my.rmd")
else:
warn("sk: Warning: scikick.yml is missing analysis field")
ymli["analysis"] = ordereddict()
else:
if ymli["analysis"] is None:
ymli["analysis"] = ordereddict()
if len(ymli["analysis"]) == 0:
if need_pages:
reterr("sk: Error: no pages have been added to scikick.yml, " + \
"this can be done with\nsk: sk add my.rmd")
else:
ymli["analysis"] = ordereddict()
if "reportdir" not in ymli.keys():
warn("sk: Warning: scikick.yml is missing reportdir field")
ymli["reportdir"] = ""
return ymli
def init_git():
"""Add certain entries to .gitignore"""
usr_dir = os.path.join(get_sk_exe_dir(), 'usr')
gitignore = ".gitignore"
anything_appended = False
if copy_file(os.path.join(usr_dir, gitignore), \
os.path.join(gitignore)):
warn("sk: File .gitignore created")
else:
existing_gitignore = open(gitignore, 'r').readlines()
template_gitignore = open(os.path.join(usr_dir, \
gitignore), 'r').readlines()
append_existing = open(gitignore, "a")
for ignore_entry in template_gitignore:
if ignore_entry.strip() not in map(str.strip, \
existing_gitignore):
append_existing.writelines(ignore_entry)
warn("sk: Added \'%s\' to .gitignore" % \
ignore_entry.rstrip())
anything_appended = True
append_existing.close()
if not anything_appended:
warn("sk: .gitignore already has all the required entries")
def add(files, deps=list(), force=False, copy_deps=None):
""" Add files and dependencies to them
files -- page file list
deps -- dependency file list
force -- bool whether to add additional index files
copy_deps -- file to copy dependencies from
"""
if deps is None:
deps = list()
ymli = yaml_in()
if copy_deps is not None:
# copy_deps(src,dest)
copy_deps = copy_deps[0]
if copy_deps not in ymli["analysis"]:
reterr(f"sk: Error: file {copy_deps} is not included")
deps2copy = ymli["analysis"][copy_deps]
if not ((deps2copy is None) or (len(deps2copy) == 0)):
deps = list(set(deps + deps2copy))
warn(f"sk: Copying {copy_deps} dependencies")
else:
warn(f"sk: Warning: {copy_deps} has no dependencies")
# add files
for fname in files:
# Add script
# Should the script be added?
add_fname = add_check(fname, ymli, force, deps)
if add_fname:
# add near scripts in the same directory
if len(ymli["analysis"].keys()) > 0:
commpath = os.path.commonpath(list(ymli["analysis"].keys()))
else:
commpath = ""
tab_name = os.path.dirname(rm_commdir(fname, commpath))
all_tabs = list(
map(lambda f: os.path.dirname(rm_commdir(f, commpath)),
ymli["analysis"].keys()))
tab_matches = list(
filter(lambda i: all_tabs[i] == tab_name,
range(len(all_tabs))))
if (tab_name != "") and (len(tab_matches) != 0):
ymli["analysis"].insert(tab_matches[-1] + 1, fname, [])
else:
ymli['analysis'][fname] = None
warn(f"sk: Added {fname}")
# Add dependencies
for dep in deps:
# Should the dep be added?
add_dep = add_dep_checks(fname, ymli, force, dep)
if add_dep:
if ymli['analysis'][fname] is None:
ymli['analysis'][fname] = []
ymli['analysis'][fname].append(dep)
warn(f"sk: Added dependency {dep} to {fname}")
if dep in ymli["analysis"].keys():
warn(
f"sk: {fname} will be executed after any executions of {dep}"
)
else:
warn(
f"sk: {fname} will be executed after any modifications to {dep}"
)
yaml_dump(ymli)
def sk_move_extras(mv_dict):
"""Performs move operations specific to scikick
Moves md files, figures in output/ directories
and renames files in scikick.yml
mv_dict -- dict with keys as src and values as dest files
"""
# Moving mds, knitmetas and output figures in out_md/;
## No need to change _site.ymls, since
## they are recreated after each change in scikick.yml
yaml_dict = yaml_in()
analysis = yaml_dict["analysis"]
reportdir = yaml_dict["reportdir"]
for src, dest in mv_dict.items():
# if not in analysis.keys -> regular file
# in this case, rename and continue
if src not in analysis.keys():
if 1 == rename(src, dest):
warn("sk: %s renamed to %s in ./scikick.yml" % (src, dest))
else:
warn("sk: Warning: %s not found in ./scikick.yml" % src)
continue
md_rootdir = os.path.join(reportdir, "out_md")
md_destdir = os.path.join(md_rootdir, os.path.dirname(dest))
md_srcdir = os.path.join(md_rootdir, os.path.dirname(src))
if not os.path.isdir(md_destdir):
os.makedirs(md_destdir)
# Move .md
md_src = os.path.join(md_rootdir, os.path.splitext(src)[0] + ".md")
md_dest = os.path.join(
md_destdir,
os.path.splitext(os.path.basename(dest))[0] + ".md")
if os.path.isfile(md_src):
shutil.move(md_src, md_dest)
# Move knitmeta
k_src = sub(pattern="\.md$", repl=".knitmeta.RDS", string=md_src)
k_dest = sub(pattern="\.md$", repl=".knitmeta.RDS", string=md_dest)
if os.path.isfile(k_src):
shutil.move(k_src, k_dest)
# Move markdown outputs (figures)
tabname_src = sub(string=os.path.basename(md_src),
pattern="\.md$",
repl="")
tabname_dest = sub(string=os.path.basename(md_dest),
pattern="\.md$",
repl="")
# "figure" must match execute_code.R fig.path
md_srcfigdir = os.path.join(md_srcdir, "figure", tabname_src)
md_destfigdir = os.path.join(md_destdir, "figure", tabname_dest)
if os.path.isdir(md_srcfigdir):
dest_dirname = os.path.dirname(md_srcfigdir)
if not os.path.isdir(dest_dirname):
os.makedirs(dest_dirname, exist_oke=True)
print(f"sk: mv {md_srcfigdir} {md_destfigdir}")
shutil.move(md_srcfigdir, md_destfigdir)
# rename the figure/ directory name in the dest md
## get the initial timestamp
initial_timestamp = os.path.getmtime(md_dest)
md_file = open(md_dest, 'r+')
md_lines = [
sub(string=line,
pattern=f'src="figure/{tabname_src}',
repl=f'src="figure/{tabname_dest}') for line in md_file
]
md_file.seek(0)
for l in md_lines:
md_file.write(l)
md_file.close()
# set the initial timestamp back to avoid reexecution
os.utime(md_dest, (initial_timestamp, initial_timestamp))
# rename all entries in scikick.yml from from to f_dest
if 1 == rename(src, dest):
warn("sk: %s renamed to %s in ./scikick.yml" % (src, dest))
else:
warn("sk: Warning: %s not found in ./scikick.yml" % src)
def sk_init(args):
"""Initialize scikick project"""
if not (args.git or args.dirs or args.yaml or args.readme or args.demo):
args.yaml = True
warn("sk: No arguments supplied, defaulting to sk init -y")
init(args)
warn("sk: See the below arguments for other " + \
"possible sk init components")
parser_init.print_help()
warn("sk: To add an R/Rmd script to the analysis use")
warn("sk: sk add my.rmd")
warn("sk: Then, to execute the workflow")
warn("sk: sk run")
else:
init(args)
def snake_status(
snakefile=get_sk_snakefile(), workdir=os.getcwd(), verbose=False,
rmd=None):
"""Print workflow status
snakefile -- string (path to the main snakefile)
workdir -- string
verbose -- bool
rmd -- string (show status for just this file)
"""
skconf = ScikickConfig(
) # OR status with no pages just indicates whether index.html exists
jobs, reasons = run_sk_dryrun(snakefile, workdir)
# split jobs/reasons into types
def subset_jobs(jobs, reasons, jtype):
# Trying to use only exe_job info
ret_jobs = []
ret_reasons = []
for i, _ in enumerate(jobs):
if job_type(jobs[i]) == jtype:
ret_jobs.append(jobs[i])
ret_reasons.append(reasons[i])
return ret_jobs, ret_reasons
exe_jobs, exe_reasons = subset_jobs(jobs, reasons, "exe_to_md")
md_jobs, md_reasons = subset_jobs(jobs, reasons, "md_to_html")
unknown_jobs, unknown_reasons = subset_jobs(jobs, reasons, "unknown")
exes = skconf.exes
if skconf.index_exe not in exes:
exes.append(skconf.index_exe)
## Parsing job/reason info from snakemake outputs
# each function matches a snakemake reason string
# get expected input updates for each script
expected_input_updates = get_expected_input_updates(
exes, exe_reasons, exe_jobs)
# get updated inputs for each script
updated_inputs = get_updated_inputs(exes, exe_reasons, exe_jobs)
# get missing outputs (which are to be generated)
missing_outs = missing_outputs(exe_reasons)
## Parsing job info only
# get which scripts will be executed (processed as exe => md)
exec_scripts = [
re.match(exe_pattern, exe_job).groups()[0] for exe_job in exe_jobs
]
# get status markers (codes ---) for each script
markers = file_markers(skconf, updated_inputs, expected_input_updates,
missing_outs, exec_scripts)
# Add site status
for job in md_jobs:
match = re.match(htmlgen_pattern, job)
# if the script has an html step fill in the gaps with ---
if match is not None:
out_base = match.groups()[0]
md_job_exe = skconf.get_info(out_base, "exe")
markers[md_job_exe] = [
"-" if x == " " else x for x in markers[md_job_exe]
]
# Checking for valid codes
# Probably better to be more careful than do this
nonexe_chars = [' ', '-']
for exe in exec_scripts:
if all([x in nonexe_chars for x in markers[exe]]):
markers[exe] = ['*', '-', '-']
warn(
f"sk: Warning: {exe} must execute for unknown reasons, (was scikick updated since last run?)"
)
# Print status or a subset
if rmd is not None:
# print only the status of rmd and dependent files
reduced_analysis = {k: skconf.analysis[k] for k in \
filter(lambda k: k in skconf.exes, \
flatten_dependency_tree(rmd, skconf))}
print_status(reduced_analysis, markers, verbose)
else:
config = skconf.analysis
# Adjustment for system index
if skconf.index_exe not in config.keys():
config['system index (homepage)'] = list() # has no inputs
markers['system index (homepage)'] = markers[skconf.index_exe]
print_status(config, markers, verbose)
def run_snakemake(snakefile=get_sk_snakefile(), workdir=os.getcwd(), \
verbose=False, dryrun=False, snakeargs=None, rmds=[], quiet=False):
"""Run snakemake with specified arguments
snakefile -- string (path to the main snakefile)
workdir -- string
verbose -- bool
dryrun -- bool
snakeargs -- list (list of additional arguments to snakemake)
rmds -- string rmd who's output should be targetted
"""
exe_dir = get_sk_exe_dir()
loghandler = os.path.join(exe_dir, 'workflow/loghandler.py')
skconf = ScikickConfig()
yml = skconf.config
# logfile created by snakemake
snake_logfile=""
### Construct call to snakemake
# before 'snakemake'
env_vars = f'SINGULARITY_BINDPATH="{exe_dir}"'
# after 'snakemake'
snakemake_args = ""
snakemake_args += f" --snakefile {snakefile}"
snakemake_args += f" --directory '{workdir}'"
snakemake_args += " --cores 1"
snakemake_args += f" --log-handler-script {loghandler}"
# Translate Rmd script to HTML target
# TODO - factor out
# TODO - move this to sk_run as an additional snake_arg
# to reduce skconfig read ins
target_arg = ""
if len(rmds) > 0:
for rmd in rmds:
# Try to add rmd if it is not in scikick.yml
if yml['analysis'] is None:
rmd_found = False
elif rmd not in yml["analysis"].keys():
rmd_found = False
else:
rmd_found = True
if not rmd_found:
warn(f"sk: Warning: {rmd} was not found in scikick.yml")
if os.path.exists(rmd):
scikick.yaml.add([rmd])
# Must reload or modify the yml since it changed
skconf = ScikickConfig()
yml = skconf.config
else:
warn(f"sk: Warning: Will not try to add non-existing file {rmd}")
return 0
# Set target. Index file is treated differently
index_rmds = get_indexes(yml)
if (len(index_rmds) == 1) and (index_rmds[0] == rmd):
target_arg += " " + os.path.join(yml["reportdir"], \
"out_html", "index.html")
else:
target_arg += " " + os.path.join(yml["reportdir"], \
"out_html", os.path.splitext(rmd)[0] + ".html")
# set more snakemake arguments
if dryrun:
snakemake_args += " --dry-run"
# Add user defined snakemake arguments
if snakeargs is not None:
snakemake_args += f" {snakeargs}"
if 'snakemake_args' in yml.keys() and yml['snakemake_args'] is not None:
warn("sk: Warning: snakemake_args is deprecated")
snakemake_args += f" {' '.join(yml['snakemake_args'])}"
# add the implied targets (html of Rmd)
snakemake_args += f" {target_arg}"
# Check for a user defined snakefile (imported by scikick Snakefile)
user_snakefile = os.path.join(os.getcwd(), "Snakefile")
if os.path.isfile(user_snakefile):
warn("sk: Including Snakefile found in project directory")
### Execution
if verbose:
warn("sk: Starting snakemake")
cmd = f"{env_vars} snakemake {snakemake_args}"
print(cmd)
sys.exit(subprocess.call(cmd, shell=True))
else:
snake_p = subprocess.Popen(f"snakemake {snakemake_args}", \
shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logs=[]
sm_err = 0
page_err = 0
while True:
line = snake_p.stdout.readline().decode('utf-8')
if not line:
break
# Capture logs
logs.append(line)
# Report progress
detect_snakemake_progress(line,quiet)
page_err = detect_page_error(line) or page_err
# In case of snakemake error start writing stderr
sm_err += detect_snakemake_error(line)
if sm_err:
sys.stderr.write(line)
logfile_name_match = re.match("^SK INTERNAL: logfile (.*)$", line)
if logfile_name_match is not None:
# Use the first found match
if snake_logfile == "":
snake_logfile = logfile_name_match.groups()[0]
snake_p.wait()
if snake_p.returncode != 0:
if not page_err:
warn("sk: Error: Snakemake returned a non-zero return code")
if not sm_err:
warn("sk: Warning: scikick was unable to find snakemake error in logs, dumping stderr...")
for line in logs:
sys.stderr.write(line)
return snake_p.returncode
else:
if not os.path.exists(skconf.homepage):
warn(f"sk: Warning: Expected homepage {skconf.homepage} is missing")
if snake_logfile != "":
rellog=os.path.relpath(snake_logfile,start=os.getcwd())
if snake_p.returncode!=0:
warn(f"sk: Complete log: {rellog}")
return snake_p.returncode
def rm(files, deps):
""" Delete files and dependencies from them
files - page file list
deps - dependency file list
"""
if deps is None:
deps = list()
ymli = yaml_in(need_pages=True)
for fname in files:
# check if rmd included
if fname not in ymli['analysis'].keys():
warn(f"sk: Warning: File {fname} not included")
continue
# delete script entry if no dependencies specified
if len(deps) == 0:
del ymli['analysis'][fname]
warn(f"sk: {fname} removed")
# Check if fname was a dependency for other scripts
for val in ymli['analysis'].values():
if val is not None:
if fname in val:
warn(
f"sk: Warning: {fname} is still a dependency of other scripts"
)
warn(
f"sk: Use sk del -d {fname} <script> to change this"
)
# delete only deps if deps specified
else:
if ymli['analysis'][fname] is None:
warn(f"sk: Warning: File {fname} has no dependencies")
continue
for dep in deps:
if dep in ymli['analysis'][fname]:
ymli['analysis'][fname].remove(dep)
warn(f"sk: dependency {dep} removed from {fname}")
else:
warn(f"sk: no dependency {dep} found for {fname}")
if len((ymli['analysis'][fname])) == 0:
ymli['analysis'][fname] = None
if os.path.splitext(os.path.basename(fname))[0] == "index":
index_list = get_indexes(ymli)
if len(index_list) == 0:
warn(f"sk: Using system template index.Rmd as homepage")
os.utime(
os.path.join(get_sk_exe_dir(), "workflow",
"notebook_rules", "index.Rmd"), None)
elif len(index_list) == 1:
os.utime(index_list[0], None)
yaml_dump(ymli)
def add_check(fname, ymli, force, deps):
"""Performs a check if fname can be added to scikick.yml as a key"""
pages = ymli["analysis"].keys()
if fname in pages:
warn(f"sk: Found existing entry for {fname}")
return False
# filenames cannot currently have wildcard symbols
if True in [i in fname for i in wildcard_symbols]:
warn(
f"sk: Error: Filename ({fname}) cannot have wildcard symbols ({' '.join(wildcard_symbols)})"
)
return False
# check if the file extension is supported
fext = os.path.splitext(fname)[-1]
if fext.lower() == ".ipynb":
warn(
"sk: Warning: .ipynb use in Scikick is experimental and requires installation of jupyter"
)
f_exe_support = fext.lower() in [x.lower() for x in supported_extensions]
if not f_exe_support:
extension_list_str = ', '.join(supported_extensions)
warn("sk: Error: Only %s files can be added as pages (%s)" % \
(extension_list_str, fname))
return False
# error if the directory doesn't exist
dirname = os.path.dirname(fname)
if dirname != "":
if not os.path.isdir(dirname):
reterr(f"sk: Error: Directory {dirname} does not exist.")
# create the file if it doesn't exist
if not os.path.isfile(fname):
warn(f"sk: Warning: File {fname} doesn't exist")
warn(f"sk: Creating new file {fname}")
open(fname, "a").close()
if fname not in pages:
# Check for other files with same basename (and therefore same md output file)
fname_base = os.path.splitext(fname)[0]
page_basenames = map(lambda x: os.path.splitext(x)[0], pages)
page_basename_exists = fname_base in page_basenames
if page_basename_exists:
warn(
f"sk: Error: Page {fname_base} is already to be compiled from another file."
)
return False
# check for "index.Rmd"s
index_list = get_indexes(ymli)
if os.path.splitext(os.path.basename(fname))[0] == "index":
if len(index_list) == 0:
warn(
f"sk: An index file {fname} has been added and will be used as the homepage"
)
# touch the added index file to ensure execution
os.utime(fname, None)
elif len(index_list) == 1:
if not force:
reterr(f"sk: Error: An index file {index_list[0]} already exists\n" + \
"sk: Error: An additional one can be added, but neither will be used as a homepage\n" + \
f"sk: Error: To persist, use 'sk add --force {fname}'")
else:
warn(
f"sk: Warning: A redundant index file has been added\n"
+
"sk: Warning: Neither of the added index files will be used as a homepage"
)
os.utime(
os.path.join(get_sk_exe_dir(), "workflow",
"notebook_rules", "index.Rmd"), None)
elif len(index_list) > 1:
warn(
f"sk: Warning: A redundant index file has been added\n" +
"sk: Warning: Neither of the added index files will be used as a homepage"
)
return True
