def loads(project=None, cmd=None, deps=[], outs=[], outs_no_cache=[], metrics_no_cache=[], fname=None, cwd=os.curdir, locked=False, add=False, overwrite=True, ignore_build_cache=False, remove_outs=False): stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from(stage, metrics_no_cache, use_cache=False, metric=True) stage.deps = dependency.loads_from(stage, deps) if fname is not None and os.path.basename(fname) != fname: msg = "Stage file name '{}' should not contain subdirectories. " \ "Use '-c|--cwd' to change location of the stage file." raise StageFileBadNameError(msg.format(fname)) fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add) Stage._check_inside_project(project, cwd) cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) stage.cwd = cwd stage.path = path # NOTE: remove outs before we check build cache if remove_outs: stage.remove_outs(ignore_remove=False) project.logger.warn("Build cache is ignored when using " "--remove-outs.") ignore_build_cache = True else: stage.unprotect_outs() if os.path.exists(path): if not ignore_build_cache and stage.is_cached(): logger.info('Stage is cached, skipping.') return None msg = "'{}' already exists. Do you wish to run the command and " \ "overwrite it?".format(stage.relpath) if not overwrite and not project.prompt.prompt(msg, False): raise StageFileAlreadyExistsError(stage.relpath) return stage
def _fill_stage_dependencies(self, **kwargs): assert not self.deps self.deps = [] self.deps += dependency.loads_from(self, kwargs.get("deps", []), erepo=kwargs.get("erepo", None)) self.deps += dependency.loads_params(self, kwargs.get("params", []))
def loads(project=None, cmd=None, deps=[], outs=[], outs_no_cache=[], metrics_no_cache=[], fname=None, cwd=os.curdir, locked=False, add=False): stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from(stage, metrics_no_cache, use_cache=False, metric=True) stage.deps = dependency.loads_from(stage, deps) fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add) cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) stage.cwd = cwd stage.path = path return stage
def get_url(url, out=None): out = resolve_output(url, out) if os.path.exists(url): url = os.path.abspath(url) out = os.path.abspath(out) dep, = dependency.loads_from(None, [url]) out, = output.loads_from(None, [out], use_cache=False) dep.download(out)
def get_url(url, out=None): out = out or os.path.basename(urlparse(url).path) if os.path.exists(url): url = os.path.abspath(url) out = os.path.abspath(out) dep, = dependency.loads_from(None, [url]) out, = output.loads_from(None, [out], use_cache=False) dep.download(out)
def get_url(url, out=None): out = resolve_output(url, out) if os.path.exists(url): url = os.path.abspath(url) out = os.path.abspath(out) (dep, ) = dependency.loads_from(None, [url]) (out, ) = output.loads_from(None, [out], use_cache=False) if not dep.exists: raise dep.DoesNotExistError(dep) dep.download(out)
def loads(project=None, cmd=None, deps=[], outs=[], outs_no_cache=[], fname=None, cwd=os.curdir): cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) stage = Stage(project=project, path=path, cmd=cmd, cwd=cwd) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.deps = dependency.loads_from(stage, deps) return stage
def get_url(url, out=None, jobs=None): import dvc.dependency as dependency import dvc.output as output from dvc.utils import resolve_output out = resolve_output(url, out) if os.path.exists(url): url = os.path.abspath(url) out = os.path.abspath(out) (dep, ) = dependency.loads_from(None, [url]) (out, ) = output.loads_from(None, [out], use_cache=False) if not dep.exists: raise dep.DoesNotExistError(dep) dep.download(out, jobs=jobs)
def loads(project=None, cmd=None, deps=[], outs=[], outs_no_cache=[], metrics_no_cache=[], fname=None, cwd=os.curdir, locked=False, add=False, overwrite=True): stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from(stage, metrics_no_cache, use_cache=False, metric=True) stage.deps = dependency.loads_from(stage, deps) if fname is not None and os.path.basename(fname) != fname: msg = "Stage file name '{}' should not contain subdirectories. " \ "Use '-c|--cwd' to change location of the stage file." raise StageFileBadNameError(msg.format(fname)) fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add) Stage._check_inside_project(project, cwd) cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) if os.path.exists(path): relpath = os.path.relpath(path) msg = "'{}' already exists. " \ "Do you wish to run the command and overwrite it?" if not overwrite \ and not project.prompt.prompt(msg.format(relpath), False): raise DvcException("'{}' already exists".format(relpath)) stage.cwd = cwd stage.path = path return stage
def loads(project=None, cmd=None, deps=[], outs=[], outs_no_cache=[], metrics_no_cache=[], fname=None, cwd=os.curdir, locked=False, add=False, overwrite=True): stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from(stage, metrics_no_cache, use_cache=False, metric=True) stage.deps = dependency.loads_from(stage, deps) fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add) cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) if os.path.exists(path): relpath = os.path.relpath(path) msg = "'{}' already exists. " \ "Do you wish to run the command and overwrite it?" if not overwrite \ and not project.prompt.prompt(msg.format(relpath), False): raise DvcException("'{}' already exists".format(relpath)) stage.cwd = cwd stage.path = path return stage
def create( repo=None, cmd=None, deps=None, outs=None, outs_no_cache=None, metrics=None, metrics_no_cache=None, fname=None, cwd=None, wdir=None, locked=False, add=False, overwrite=True, ignore_build_cache=False, remove_outs=False, validate_state=True, outs_persist=None, outs_persist_no_cache=None, erepo=None, ): if outs is None: outs = [] if deps is None: deps = [] if outs_no_cache is None: outs_no_cache = [] if metrics is None: metrics = [] if metrics_no_cache is None: metrics_no_cache = [] if outs_persist is None: outs_persist = [] if outs_persist_no_cache is None: outs_persist_no_cache = [] # Backward compatibility for `cwd` option if wdir is None and cwd is not None: if fname is not None and os.path.basename(fname) != fname: raise StageFileBadNameError( "stage file name '{fname}' may not contain subdirectories" " if '-c|--cwd' (deprecated) is specified. Use '-w|--wdir'" " along with '-f' to specify stage file path and working" " directory.".format(fname=fname) ) wdir = cwd else: wdir = os.curdir if wdir is None else wdir stage = Stage(repo=repo, wdir=wdir, cmd=cmd, locked=locked) Stage._fill_stage_outputs( stage, outs, outs_no_cache, metrics, metrics_no_cache, outs_persist, outs_persist_no_cache, ) stage.deps = dependency.loads_from(stage, deps, erepo=erepo) stage._check_circular_dependency() stage._check_duplicated_arguments() if not fname: fname = Stage._stage_fname(stage.outs, add=add) stage._check_dvc_filename(fname) wdir = os.path.abspath(wdir) if cwd is not None: path = os.path.join(wdir, fname) else: path = os.path.abspath(fname) Stage._check_stage_path(repo, wdir) Stage._check_stage_path(repo, os.path.dirname(path)) stage.wdir = wdir stage.path = path # NOTE: remove outs before we check build cache if remove_outs: logger.warning( "--remove-outs is deprecated." " It is now the default behavior," " so there's no need to use this option anymore." ) stage.remove_outs(ignore_remove=False) logger.warning("Build cache is ignored when using --remove-outs.") ignore_build_cache = True else: stage.unprotect_outs() if os.path.exists(path) and any(out.persist for out in stage.outs): logger.warning("Build cache is ignored when persisting outputs.") ignore_build_cache = True if validate_state: if os.path.exists(path): if not ignore_build_cache and stage.is_cached: logger.info("Stage is cached, skipping.") return None msg = ( "'{}' already exists. Do you wish to run the command and " "overwrite it?".format(stage.relpath) ) if not overwrite and not prompt.confirm(msg): raise StageFileAlreadyExistsError(stage.relpath) os.unlink(path) return stage
def fill_stage_dependencies(stage, deps=None, erepo=None, params=None): assert not stage.deps stage.deps = [] stage.deps += dependency.loads_from(stage, deps or [], erepo=erepo) stage.deps += dependency.loads_params(stage, params or [])
def create(repo, **kwargs): wdir = kwargs.get("wdir", None) cwd = kwargs.get("cwd", None) fname = kwargs.get("fname", None) add = kwargs.get("add", False) # Backward compatibility for `cwd` option if wdir is None and cwd is not None: if fname is not None and os.path.basename(fname) != fname: raise StageFileBadNameError( "DVC-file name '{fname}' may not contain subdirectories" " if `-c|--cwd` (deprecated) is specified. Use `-w|--wdir`" " along with `-f` to specify DVC-file path with working" " directory.".format(fname=fname)) wdir = cwd elif wdir is None: wdir = os.curdir stage = Stage( repo=repo, wdir=wdir, cmd=kwargs.get("cmd", None), locked=kwargs.get("locked", False), always_changed=kwargs.get("always_changed", False), ) Stage._fill_stage_outputs(stage, **kwargs) stage.deps = dependency.loads_from(stage, kwargs.get("deps", []), erepo=kwargs.get("erepo", None)) stage._check_circular_dependency() stage._check_duplicated_arguments() if not fname: fname = Stage._stage_fname(stage.outs, add) stage._check_dvc_filename(fname) # Autodetecting wdir for add, we need to create outs first to do that, # so we start with wdir = . and remap out paths later. if add and kwargs.get("wdir") is None and cwd is None: wdir = os.path.dirname(fname) for out in chain(stage.outs, stage.deps): if out.is_in_repo: out.def_path = relpath(out.path_info, wdir) wdir = os.path.abspath(wdir) if cwd is not None: path = os.path.join(wdir, fname) else: path = os.path.abspath(fname) Stage._check_stage_path(repo, wdir) Stage._check_stage_path(repo, os.path.dirname(path)) stage.wdir = wdir stage.path = path ignore_build_cache = kwargs.get("ignore_build_cache", False) # NOTE: remove outs before we check build cache if kwargs.get("remove_outs", False): logger.warning("--remove-outs is deprecated." " It is now the default behavior," " so there's no need to use this option anymore.") stage.remove_outs(ignore_remove=False) logger.warning("Build cache is ignored when using --remove-outs.") ignore_build_cache = True if os.path.exists(path) and any(out.persist for out in stage.outs): logger.warning("Build cache is ignored when persisting outputs.") ignore_build_cache = True if os.path.exists(path): if (not ignore_build_cache and stage.is_cached and not stage.is_callback and not stage.always_changed): logger.info("Stage is cached, skipping.") return None msg = ("'{}' already exists. Do you wish to run the command and " "overwrite it?".format(stage.relpath)) if not kwargs.get("overwrite", True) and not prompt.confirm(msg): raise StageFileAlreadyExistsError(stage.relpath) os.unlink(path) return stage
def create( repo=None, cmd=None, deps=None, outs=None, outs_no_cache=None, metrics=None, metrics_no_cache=None, fname=None, cwd=os.curdir, locked=False, add=False, overwrite=True, ignore_build_cache=False, remove_outs=False, ): if outs is None: outs = [] if deps is None: deps = [] if outs_no_cache is None: outs_no_cache = [] if metrics is None: metrics = [] if metrics_no_cache is None: metrics_no_cache = [] stage = Stage(repo=repo, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from( stage, metrics, use_cache=True, metric=True ) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from( stage, metrics_no_cache, use_cache=False, metric=True ) stage.deps = dependency.loads_from(stage, deps) stage._check_circular_dependency() stage._check_duplicated_arguments() if fname is not None and os.path.basename(fname) != fname: raise StageFileBadNameError( "stage file name '{fname}' should not contain subdirectories." " Use '-c|--cwd' to change location of the stage file.".format( fname=fname ) ) fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add) Stage._check_inside_repo(repo, cwd) cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) stage.cwd = cwd stage.path = path # NOTE: remove outs before we check build cache if remove_outs: stage.remove_outs(ignore_remove=False) logger.warning("Build cache is ignored when using --remove-outs.") ignore_build_cache = True else: stage.unprotect_outs() if os.path.exists(path): if not ignore_build_cache and stage.is_cached: logger.info("Stage is cached, skipping.") return None msg = ( "'{}' already exists. Do you wish to run the command and " "overwrite it?".format(stage.relpath) ) if not overwrite and not prompt.confirm(msg): raise StageFileAlreadyExistsError(stage.relpath) os.unlink(path) return stage
def _load_deps(cls, stage, data): stage.deps.extend(dependency.loads_from(stage, data))