def abs_file_paths(xs, base_dir=None, ignore_keys=None): """Normalize any file paths found in a subdirectory of configuration input. """ ignore_keys = set([]) if ignore_keys is None else set(ignore_keys) if base_dir is None: base_dir = os.getcwd() orig_dir = os.getcwd() os.chdir(base_dir) input_dir = os.path.join(base_dir, "inputs") if isinstance(xs, dict): out = {} for k, v in xs.iteritems(): if k not in ignore_keys and v and isinstance(v, basestring): if v.lower() == "none": out[k] = None elif os.path.exists(v) or objectstore.is_remote(v): out[k] = os.path.normpath( os.path.join(base_dir, objectstore.download(v, input_dir))) else: out[k] = v else: out[k] = v elif isinstance(xs, basestring): if os.path.exists(xs) or objectstore.is_remote(xs): out = os.path.normpath( os.path.join(base_dir, objectstore.download(xs, input_dir))) else: out = xs else: out = xs os.chdir(orig_dir) return out
def abs_file_paths(xs, base_dir=None, ignore_keys=None): """Normalize any file paths found in a subdirectory of configuration input. """ ignore_keys = set([]) if ignore_keys is None else set(ignore_keys) if base_dir is None: base_dir = os.getcwd() orig_dir = os.getcwd() os.chdir(base_dir) input_dir = os.path.join(base_dir, "inputs") if isinstance(xs, dict): out = {} for k, v in xs.iteritems(): if k not in ignore_keys and v and isinstance(v, basestring): if v.lower() == "none": out[k] = None elif os.path.exists(v) or objectstore.is_remote(v): out[k] = os.path.normpath(os.path.join(base_dir, objectstore.download(v, input_dir))) else: out[k] = v else: out[k] = v elif isinstance(xs, basestring): if os.path.exists(xs) or objectstore.is_remote(xs): out = os.path.normpath(os.path.join(base_dir, objectstore.download(xs, input_dir))) else: out = xs else: out = xs os.chdir(orig_dir) return out
def abs_file_paths(xs, base_dir=None, ignore_keys=None, fileonly_keys=None, cur_key=None, do_download=True): """Normalize any file paths found in a subdirectory of configuration input. base_dir -- directory to normalize relative paths to ignore_keys -- algorithm key names to ignore normalize for (keywords, not files/directories) fileonly_keys -- algorithm key names to only expand files (not directories) cur_key -- current key when calling recursively """ ignore_keys = set([]) if ignore_keys is None else set(ignore_keys) fileonly_keys = set([]) if fileonly_keys is None else set(fileonly_keys) if base_dir is None: base_dir = os.getcwd() orig_dir = os.getcwd() os.chdir(base_dir) input_dir = os.path.join(base_dir, "inputs") if isinstance(xs, dict): out = {} for k, v in xs.items(): if k not in ignore_keys and v and isinstance(v, basestring): if v.lower() == "none": out[k] = None else: out[k] = abs_file_paths(v, base_dir, ignore_keys, fileonly_keys, k, do_download=do_download) elif isinstance(v, (list, tuple)): out[k] = [ abs_file_paths(x, base_dir, ignore_keys, fileonly_keys, k, do_download=do_download) for x in v ] else: out[k] = v elif isinstance(xs, basestring): if os.path.exists(xs) or (do_download and objectstore.is_remote(xs)): dl = objectstore.download(xs, input_dir) if dl and cur_key not in ignore_keys and not ( cur_key in fileonly_keys and not os.path.isfile(dl)): out = os.path.normpath(os.path.join(base_dir, dl)) else: out = xs else: out = xs else: out = xs os.chdir(orig_dir) return out
def abs_file_paths(xs, base_dir=None, ignore_keys=None, fileonly_keys=None, cur_key=None, do_download=True): """Normalize any file paths found in a subdirectory of configuration input. base_dir -- directory to normalize relative paths to ignore_keys -- algorithm key names to ignore normalize for (keywords, not files/directories) fileonly_keys -- algorithm key names to only expand files (not directories) cur_key -- current key when calling recursively """ ignore_keys = set([]) if ignore_keys is None else set(ignore_keys) fileonly_keys = set([]) if fileonly_keys is None else set(fileonly_keys) if base_dir is None: base_dir = os.getcwd() orig_dir = os.getcwd() os.chdir(base_dir) input_dir = os.path.join(base_dir, "inputs") if isinstance(xs, dict): out = {} for k, v in xs.items(): if k not in ignore_keys and v and isinstance(v, six.string_types): if v.lower() == "none": out[k] = None else: out[k] = abs_file_paths(v, base_dir, ignore_keys, fileonly_keys, k, do_download=do_download) elif isinstance(v, (list, tuple)): out[k] = [abs_file_paths(x, base_dir, ignore_keys, fileonly_keys, k, do_download=do_download) for x in v] else: out[k] = v elif isinstance(xs, six.string_types): if os.path.exists(xs) or (do_download and objectstore.is_remote(xs)): dl = objectstore.download(xs, input_dir) if dl and cur_key not in ignore_keys and not (cur_key in fileonly_keys and not os.path.isfile(dl)): out = os.path.normpath(os.path.join(base_dir, dl)) else: out = xs else: out = xs else: out = xs os.chdir(orig_dir) return out
def _add_remote_resources(resources): """Retrieve remote resources like GATK/MuTect jars present in S3. """ out = copy.deepcopy(resources) for prog, info in resources.iteritems(): for key, val in info.iteritems(): if key == "jar" and objectstore.is_remote(val): store_dir = utils.safe_makedir(os.path.join(os.getcwd(), "inputs", "jars", prog)) fname = objectstore.download(val, store_dir, store_dir) version_file = os.path.join(store_dir, "version.txt") if not utils.file_exists(version_file): version = install.get_gatk_jar_version(prog, fname) with open(version_file, "w") as out_handle: out_handle.write(version) else: with open(version_file) as in_handle: version = in_handle.read().strip() del out[prog][key] out[prog]["dir"] = store_dir out[prog]["version"] = version return out
def _add_remote_resources(resources): """Retrieve remote resources like GATK/MuTect jars present in S3. """ out = copy.deepcopy(resources) for prog, info in resources.items(): for key, val in info.items(): if key == "jar" and objectstore.is_remote(val): store_dir = utils.safe_makedir(os.path.join(os.getcwd(), "inputs", "jars", prog)) fname = objectstore.download(val, store_dir, store_dir) version_file = os.path.join(store_dir, "version.txt") if not utils.file_exists(version_file): version = install.get_gatk_jar_version(prog, fname) with open(version_file, "w") as out_handle: out_handle.write(version) else: with open(version_file) as in_handle: version = in_handle.read().strip() del out[prog][key] out[prog]["dir"] = store_dir out[prog]["version"] = version return out