def configfile(self, fp):
""" Update the global config with data from the given file. """
global config
self.configfiles.append(fp)
c = snakemake.io.load_configfile(fp)
update_config(config, c)
update_config(config, self.overwrite_config)
def configfile(self, jsonpath):
""" Update the global config with the given dictionary. """
global config
self.configfiles.append(jsonpath)
c = snakemake.io.load_configfile(jsonpath)
update_config(config, c)
update_config(config, self.overwrite_config)
def test_update_config():
a = {'foo':{'bar':{'foo': 'bar', 'bar':'foo'}}}
b = {'foo':{'bar':{'foo': 'barfoo'}}}
update_config(a, b)
c = a
update_config(b, a)
d = b
assert c == {'foo': {'bar': {'foo': 'barfoo', 'bar': 'foo'}}}
assert d == {'foo': {'bar': {'bar': 'foo', 'foo': 'barfoo'}}}
def Init(argv):
parser = argparse.ArgumentParser(
"mars init",
description=
"Creates a config file for MARS, optionally populated with values")
parser.add_argument(
'-o',
'--output',
help="Output file to write config file (default: stdout)")
parser.add_argument(
'-c',
'--configfile',
type=argparse.FileType('r'),
help=
"Path to another config file whose values will be added to this one (where possible)"
)
parser.add_argument('--force',
action='store_true',
help="Overwite output file if it exists")
parser.add_argument(
"values",
nargs=argparse.REMAINDER,
help=("Config values to be added to the config file in format "
"`key:value`. Overrides values from other config file if both "
"present."))
args = parser.parse_args(argv)
if args.output and Path(args.output).exists() and not args.force:
mars_error("Chosen output file exists. Use --force to overwrite.")
elif args.output:
output = open(args.output, 'w')
else:
output = sys.stdout
old_config = yaml.safe_load(args.configfile) if args.configfile else {}
# Loading a blank file will return None from yaml.safe_load()
if old_config is None:
old_config = {}
cmdline_kv = {}
for kv_pair in args.values:
try:
key, value = kv_pair.strip().split(":")
cmdline_kv[key] = value
except ValueError as e:
mars_error("Could not parse key:value '{}': {}".format(kv_pair, e))
update_config(old_config, cmdline_kv)
config, unused_keys = create_config(**old_config)
output.write(config)
if unused_keys:
logger.warn(
"Warning: the following keys were specified but unused: {}".format(
unused_keys))
logger.info(
"Config values not specified are commented out in the config file. "
"Uncomment the relevant lines and add appropriate values as necessary."
)
def update_config(config):
"""
Populates config file with default config values.
And made changes if necessary.
"""
# get default values and update them with values specified in config file
default_config = make_default_config()
utils.update_config(default_config, config)
return default_config
def update_config(config):
"""
Populates config file with default config values.
And made changes if necessary.
"""
# in old version java_mem was used, new is mem
if ('java_mem' in config) and (not ('mem' in config)):
config['mem'] = config['java_mem']
# get default values and update them with values specified in config file
default_config = make_default_config()
utils.update_config(default_config, config)
return default_config
def post(self, name):
# get data from post
data = request.get_json()
pipeline = PipelineModel.find_by_name(name)
if not pipeline:
return {'message': f"Pipeline {name} not found in Database"}
# load config json into config dict
path_to_snakefile = os.path.join(path_to_static, name, 'Snakefile')
# load user configs if given
config = pipeline.json()['config']
if data:
user_config = data['config']
# use the function load_config from snakemake itself!!
update_config(config, user_config)
workdir = config['workdir']
os.makedirs(workdir, exist_ok=True)
path_to_config_file = os.path.join(workdir, 'config.yaml')
path_to_dag = os.path.join(workdir, 'dag.json')
with open(path_to_config_file, 'w+') as stream:
yaml.dump(config, stream, default_flow_style=False)
try:
dagcmd = f"snakemake --snakefile {path_to_snakefile} --configfile {path_to_config_file} --d3dag > {path_to_dag}"
# ! place dag in d3dag and return
snakedag = Popen(dagcmd,
shell=True,
stdout=PIPE,
stderr=STDOUT,
close_fds=True)
except CalledProcessError as er:
print(f'Snakemake returned Error code {er.returncode}')
snakedag.communicate()
with open(path_to_dag, 'r') as file:
d3dag = json.load(file)
info = {'dag': d3dag, 'config': config}
# run the snakemake jobs
# try:
# subprocess.check_call(['snakemake', '--snakefile', path_to_snakefile, '--configfile', path_to_config_file])
# except subprocess.CalledProcessError as er:
# print(f'Snakemake returned Error code {er.returncode}')
return info
def register_metadata(metadata_file, config):
"""Read an SRA project file and register metadata in sml_config. Will
issue a warning if file does not exists.
Args:
metadata - file name
config - configuration to update
"""
metadata_list = []
import sys
if metadata_file in sys.argv:
return config
try:
with open(metadata_file, "r") as fh:
reader = csv.DictReader(fh.readlines())
metadata_list = [row for row in reader]
run2sample = {row["Run"]:row["SampleName"] for row in metadata_list}
config_default = {
'bio.ngs.settings' : {
'sampleinfo' : metadata_file
},
'bio.ngs.tools.sratools': {
'_datadir': os.path.dirname(metadata_file),
'_run2sample' : run2sample,
'_metadata' : metadata_list
},
}
update_config(config_default, config)
config = config_default
except Exception:
raise Exception("""
no metadata file '{metadata}' found
please initiate analysis by running 'snakemake {metadata}'
""".format(metadata=metadata_file))
return config
def check_config_default(c):
for fn in [c['config_default']]:
assert op.isfile(fn), "cannot read %s" % fn
cfg_default = yaml.safe_load(open(c['config_default'], 'r'))
update_config(cfg_default, c)
c = cfg_default
for fn in [c['config_job_default']]:
assert op.isfile(fn), "cannot read %s" % fn
cfg_job = read_job_config(c)
update_config(cfg_job, c)
c = cfg_job
dirh0, dirc0 = c['dir_project'], c['dir_cache']
pid, wid, oid = c['pid'], c['wid'], c['oid']
c['dirh'] = op.join(dirh0, pid, wid)
c['dirc'] = op.join(dirc0, pid, wid)
c['dirr'] = op.join(dirh0, pid, wid, oid)
dirh, dirc, dirr = c['dirh'], c['dirc'], c['dirr']
dirr_l = op.join(dirc, oid)
for subdir in [dirh, dirc, dirr, c['tmpdir']]:
if not op.isdir(subdir):
makedirs(subdir)
make_symlink(dirr, dirr_l)
dirh_l = op.join(dirc, 'primary')
dirc_l = op.join(dirh, 'cache')
make_symlink(dirc, dirc_l)
make_symlink(dirh, dirh_l)
xdic = read_genome_config(c)
gdic = {g: dict() for g in xdic.keys()}
c['x'] = xdic
c['g'] = gdic
return c
ap = argparse.ArgumentParser()
ap.add_argument('config', help='Main config.yaml file')
ap.add_argument('hub_config', help='Track hub config YAML file')
ap.add_argument('--additional-configs',
nargs='+',
help='Additional config files with which to update the main '
'config')
args = ap.parse_args()
# Access configured options. See comments in example hub_config.yaml for
# details
config = yaml.load(open(args.config), Loader=yaml.FullLoader)
if args.additional_configs:
for cfg in args.additional_configs:
update_config(config, yaml.load(open(cfg), Loader=yaml.FullLoader))
hub_config = yaml.load(open(args.hub_config), Loader=yaml.FullLoader)
hub, genomes_file, genome, trackdb = default_hub(
hub_name=hub_config['hub']['name'],
short_label=hub_config['hub']['short_label'],
long_label=hub_config['hub']['long_label'],
email=hub_config['hub']['email'],
genome=hub_config['hub']['genome'])
c = ChIPSeqConfig(
config, os.path.join(os.path.dirname(args.config),
'chipseq_patterns.yaml'))
# Set up subgroups based on unique values from columns specified in the config
def configfile(self, jsonpath):
""" Update the global config with the given dictionary. """
global config
c = snakemake.io.load_configfile(jsonpath)
update_config(config, c)
update_config(config, self.overwrite_config)
def test_update_config_with_string():
with pytest.raises(AttributeError):
update_config({}, "foo")
def update_config(config_new):
utils.update_config("config.yaml", config_new)
return
ap = argparse.ArgumentParser()
ap.add_argument('config', help='Main config.yaml file')
ap.add_argument('hub_config', help='Track hub config YAML file')
ap.add_argument('--additional-configs',
nargs='+',
help='Additional config files with which to update the main '
'config')
args = ap.parse_args()
# Access configured options. See comments in example hub_config.yaml for
# details
config = yaml.load(open(args.config))
if args.additional_configs:
for cfg in args.additional_configs:
update_config(config, yaml.load(open(cfg)))
hub_config = yaml.load(open(args.hub_config))
hub, genomes_file, genome, trackdb = default_hub(
hub_name=hub_config['hub']['name'],
short_label=hub_config['hub']['short_label'],
long_label=hub_config['hub']['long_label'],
email=hub_config['hub']['email'],
genome=hub_config['hub']['genome'])
c = ChIPSeqConfig(
config, os.path.join(os.path.dirname(args.config),
'chipseq_patterns.yaml'))
# Set up subgroups based on unique values from columns specified in the config
def Init(argv):
parser = argparse.ArgumentParser(
"venus init",
description="Creates config and samplesheet files for VENUS")
parser.add_argument(
'-o',
'--output',
default="config.yml",
help="Output file to write config file (default: config.yml)")
parser.add_argument(
'-c',
'--configfile',
type=argparse.FileType('r'),
help=
"Path to another config file whose values will be added to this one (where possible)"
)
parser.add_argument('--force',
action='store_true',
help="Overwite output file if it exists")
parser.add_argument(
"values",
nargs=argparse.REMAINDER,
default=["output_dir:venus_output", "samplesheet_fp:samplesheet.tsv"],
help=("Config values to be added to the config file in format "
"`key:value`. Overrides values from other config file if both "
"present. (default: %(default)s)"))
args = parser.parse_args(argv)
if Path(args.output).exists() and not args.force:
venus_error(
"Output file '{}' exists. Use --force to overwrite.".format(
args.output))
else:
output = open(args.output, 'w')
old_config = yaml.safe_load(args.configfile) if args.configfile else {}
# Loading a blank file will return None from yaml.safe_load()
if old_config is None:
old_config = {}
cmdline_kv = {}
if not args.values:
args.values = [
"output_dir:venus_output", "samplesheet_fp:samplesheet.tsv"
]
logger.info("Default output_dir name chosen: 'venus_output'")
logger.info("Default samplesheet_fp chosen: 'samplesheet.tsv'")
logger.info("Edit these if they don't match your setup!")
for kv_pair in args.values:
try:
key, value = kv_pair.strip().split(":")
cmdline_kv[key] = value
except ValueError as e:
venus_error("Could not parse key:value '{}': {}".format(
kv_pair, e))
update_config(old_config, cmdline_kv)
config, unused_keys = create_config(**old_config)
output.write(config)
if unused_keys:
logger.warn(
"Warning: the following keys were specified but unused: {}".format(
unused_keys))
logger.info(
"Config values not specified are commented out in the config file. "
"Uncomment the relevant lines and add appropriate values as necessary."
)
logger.info("** Config file written to {} **".format(args.output))
default_config_sections = ["db", "quant", "filter", "analysis", "qc", "bfq", "samples"]
for section in default_config_sections:
if section not in config:
config[section] = {}
# default config
main_fn = srcdir("main.config")
with open(main_fn) as fh:
CONF = yaml.load(fh, Loader=Loader) or {}
GCF_SECRET = os.environ.get("GCF_SECRET")
if GCF_SECRET:
with open(GCF_SECRET) as fh:
SECRETS = yaml.load(fh, Loader=Loader) or {}
update_config(CONF, SECRETS)
# library preparation kit specific configuration
libprep_fn = srcdir("libprep.config")
with open(libprep_fn) as fh:
LIBPREP_CONF = yaml.load(fh, Loader=Loader) or {}
kit = config.get("libprepkit")
if kit is not None:
if len(config["read_geometry"]) > 1:
kit += " PE"
else:
kit += " SE"
if kit in LIBPREP_CONF:
# overwrite default config
update_config(CONF, LIBPREP_CONF[kit])
else:
