def abs_file_paths(xs, base_dir=None, ignore_keys=None):
"""Normalize any file paths found in a subdirectory of configuration input.
"""
ignore_keys = set([]) if ignore_keys is None else set(ignore_keys)
if base_dir is None:
base_dir = os.getcwd()
orig_dir = os.getcwd()
os.chdir(base_dir)
input_dir = os.path.join(base_dir, "inputs")
if isinstance(xs, dict):
out = {}
for k, v in xs.iteritems():
if k not in ignore_keys and v and isinstance(v, basestring):
if v.lower() == "none":
out[k] = None
elif os.path.exists(v) or objectstore.is_remote(v):
out[k] = os.path.normpath(
os.path.join(base_dir,
objectstore.download(v, input_dir)))
else:
out[k] = v
else:
out[k] = v
elif isinstance(xs, basestring):
if os.path.exists(xs) or objectstore.is_remote(xs):
out = os.path.normpath(
os.path.join(base_dir, objectstore.download(xs, input_dir)))
else:
out = xs
else:
out = xs
os.chdir(orig_dir)
return out
def abs_file_paths(xs, base_dir=None, ignore_keys=None):
"""Normalize any file paths found in a subdirectory of configuration input.
"""
ignore_keys = set([]) if ignore_keys is None else set(ignore_keys)
if base_dir is None:
base_dir = os.getcwd()
orig_dir = os.getcwd()
os.chdir(base_dir)
input_dir = os.path.join(base_dir, "inputs")
if isinstance(xs, dict):
out = {}
for k, v in xs.iteritems():
if k not in ignore_keys and v and isinstance(v, basestring):
if v.lower() == "none":
out[k] = None
elif os.path.exists(v) or objectstore.is_remote(v):
out[k] = os.path.normpath(os.path.join(base_dir, objectstore.download(v, input_dir)))
else:
out[k] = v
else:
out[k] = v
elif isinstance(xs, basestring):
if os.path.exists(xs) or objectstore.is_remote(xs):
out = os.path.normpath(os.path.join(base_dir, objectstore.download(xs, input_dir)))
else:
out = xs
else:
out = xs
os.chdir(orig_dir)
return out
def abs_file_paths(xs,
base_dir=None,
ignore_keys=None,
fileonly_keys=None,
cur_key=None,
do_download=True):
"""Normalize any file paths found in a subdirectory of configuration input.
base_dir -- directory to normalize relative paths to
ignore_keys -- algorithm key names to ignore normalize for (keywords, not files/directories)
fileonly_keys -- algorithm key names to only expand files (not directories)
cur_key -- current key when calling recursively
"""
ignore_keys = set([]) if ignore_keys is None else set(ignore_keys)
fileonly_keys = set([]) if fileonly_keys is None else set(fileonly_keys)
if base_dir is None:
base_dir = os.getcwd()
orig_dir = os.getcwd()
os.chdir(base_dir)
input_dir = os.path.join(base_dir, "inputs")
if isinstance(xs, dict):
out = {}
for k, v in xs.items():
if k not in ignore_keys and v and isinstance(v, basestring):
if v.lower() == "none":
out[k] = None
else:
out[k] = abs_file_paths(v,
base_dir,
ignore_keys,
fileonly_keys,
k,
do_download=do_download)
elif isinstance(v, (list, tuple)):
out[k] = [
abs_file_paths(x,
base_dir,
ignore_keys,
fileonly_keys,
k,
do_download=do_download) for x in v
]
else:
out[k] = v
elif isinstance(xs, basestring):
if os.path.exists(xs) or (do_download and objectstore.is_remote(xs)):
dl = objectstore.download(xs, input_dir)
if dl and cur_key not in ignore_keys and not (
cur_key in fileonly_keys and not os.path.isfile(dl)):
out = os.path.normpath(os.path.join(base_dir, dl))
else:
out = xs
else:
out = xs
else:
out = xs
os.chdir(orig_dir)
return out
def abs_file_paths(xs, base_dir=None, ignore_keys=None, fileonly_keys=None, cur_key=None,
do_download=True):
"""Normalize any file paths found in a subdirectory of configuration input.
base_dir -- directory to normalize relative paths to
ignore_keys -- algorithm key names to ignore normalize for (keywords, not files/directories)
fileonly_keys -- algorithm key names to only expand files (not directories)
cur_key -- current key when calling recursively
"""
ignore_keys = set([]) if ignore_keys is None else set(ignore_keys)
fileonly_keys = set([]) if fileonly_keys is None else set(fileonly_keys)
if base_dir is None:
base_dir = os.getcwd()
orig_dir = os.getcwd()
os.chdir(base_dir)
input_dir = os.path.join(base_dir, "inputs")
if isinstance(xs, dict):
out = {}
for k, v in xs.items():
if k not in ignore_keys and v and isinstance(v, six.string_types):
if v.lower() == "none":
out[k] = None
else:
out[k] = abs_file_paths(v, base_dir, ignore_keys, fileonly_keys, k, do_download=do_download)
elif isinstance(v, (list, tuple)):
out[k] = [abs_file_paths(x, base_dir, ignore_keys, fileonly_keys, k, do_download=do_download)
for x in v]
else:
out[k] = v
elif isinstance(xs, six.string_types):
if os.path.exists(xs) or (do_download and objectstore.is_remote(xs)):
dl = objectstore.download(xs, input_dir)
if dl and cur_key not in ignore_keys and not (cur_key in fileonly_keys and not os.path.isfile(dl)):
out = os.path.normpath(os.path.join(base_dir, dl))
else:
out = xs
else:
out = xs
else:
out = xs
os.chdir(orig_dir)
return out
def _add_remote_resources(resources):
"""Retrieve remote resources like GATK/MuTect jars present in S3.
"""
out = copy.deepcopy(resources)
for prog, info in resources.iteritems():
for key, val in info.iteritems():
if key == "jar" and objectstore.is_remote(val):
store_dir = utils.safe_makedir(os.path.join(os.getcwd(), "inputs", "jars", prog))
fname = objectstore.download(val, store_dir, store_dir)
version_file = os.path.join(store_dir, "version.txt")
if not utils.file_exists(version_file):
version = install.get_gatk_jar_version(prog, fname)
with open(version_file, "w") as out_handle:
out_handle.write(version)
else:
with open(version_file) as in_handle:
version = in_handle.read().strip()
del out[prog][key]
out[prog]["dir"] = store_dir
out[prog]["version"] = version
return out
def _add_remote_resources(resources):
"""Retrieve remote resources like GATK/MuTect jars present in S3.
"""
out = copy.deepcopy(resources)
for prog, info in resources.items():
for key, val in info.items():
if key == "jar" and objectstore.is_remote(val):
store_dir = utils.safe_makedir(os.path.join(os.getcwd(), "inputs", "jars", prog))
fname = objectstore.download(val, store_dir, store_dir)
version_file = os.path.join(store_dir, "version.txt")
if not utils.file_exists(version_file):
version = install.get_gatk_jar_version(prog, fname)
with open(version_file, "w") as out_handle:
out_handle.write(version)
else:
with open(version_file) as in_handle:
version = in_handle.read().strip()
del out[prog][key]
out[prog]["dir"] = store_dir
out[prog]["version"] = version
return out
