def run(self):
recursive = not self.args.single_item
saved_dir = os.path.realpath(os.curdir)
if self.args.cwd:
os.chdir(self.args.cwd)
# Dirty hack so the for loop below can at least enter once
if self.args.all_pipelines:
self.args.targets = [None]
ret = 0
for target in self.args.targets:
try:
stages = self.project.reproduce(
target,
recursive=recursive,
force=self.args.force,
dry=self.args.dry,
interactive=self.args.interactive,
pipeline=self.args.pipeline,
all_pipelines=self.args.all_pipelines,
ignore_build_cache=self.args.ignore_build_cache)
if len(stages) == 0:
logger.info(CmdDataStatus.UP_TO_DATE_MSG)
if self.args.metrics:
self.project.metrics_show()
except DvcException:
logger.error()
ret = 1
break
os.chdir(saved_dir)
return ret
def test_traceback(self):
stack_trace1 = "stack_trace1\n"
stack_trace2 = "stack_trace2\n"
try:
exc1 = Exception("exception1")
exc2 = DvcException("exception2", cause=exc1)
exc2.cause_tb = stack_trace1
exc3 = DvcException("exception3", cause=exc2)
exc3.cause_tb = stack_trace2
raise exc3
except Exception:
stack_trace3 = traceback.format_exc()
with logger.verbose():
logger.error("message")
output = ("Error: message - exception3: exception2: exception1\n"
"{line}\n"
"{stack_trace1}"
"\n"
"{stack_trace2}"
"\n"
"{stack_trace3}"
"{line}\n"
"\n"
"Having any troubles? Hit us up at https://dvc.org/support,"
" we are always happy to help!\n").format(
line="-" * 60,
stack_trace1=stack_trace1,
stack_trace2=stack_trace2,
stack_trace3=stack_trace3,
)
self.assertEqual(self.stderr, output)
def run(self):
from dvc.remote import _get, RemoteLOCAL
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
if remote == RemoteLOCAL:
self.args.url = self.resolve_path(
self.args.url, self.configobj.filename
)
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
if (section in self.configobj.keys()) and not self.args.force:
logger.error(
"Remote with name {} already exists. "
"Use -f (--force) to overwrite remote "
"with new value".format(self.args.name)
)
return 1
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
if ret != 0:
return ret
if self.args.default:
msg = "Setting '{}' as a default remote.".format(self.args.name)
logger.info(msg)
ret = self._set(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
)
return ret
def load_dir_cache(self, md5):
path = self.get(md5)
assert self.is_dir_cache(path)
dir_info = self._dir_info.get(md5)
if dir_info:
return dir_info
try:
with open(path, "r") as fd:
d = json.load(fd)
except Exception:
msg = "Failed to load dir cache '{}'"
logger.error(msg.format(os.path.relpath(path)))
return []
if not isinstance(d, list):
msg = "dir cache file format error '{}' [skipping the file]"
logger.error(msg.format(os.path.relpath(path)))
return []
for info in d:
info["relpath"] = self.to_ospath(info["relpath"])
self._dir_info[md5] = d
return d
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
s3 = self.s3
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info["scheme"] != "s3":
raise NotImplementedError
if from_info["scheme"] != "local":
raise NotImplementedError
logger.debug("Uploading '{}' to '{}/{}'".format(
from_info["path"], to_info["bucket"], to_info["path"]))
if not name:
name = os.path.basename(from_info["path"])
total = os.path.getsize(from_info["path"])
cb = Callback(name, total)
try:
s3.upload_file(from_info["path"],
to_info["bucket"],
to_info["path"],
Callback=cb)
except Exception:
msg = "failed to upload '{}'".format(from_info["path"])
logger.error(msg)
continue
progress.finish_target(name)
def do_run(self, target=None):
indent = 1 if self.args.cloud else 0
try:
st = self.project.status(
target=target,
jobs=self.args.jobs,
cloud=self.args.cloud,
show_checksums=self.args.show_checksums,
remote=self.args.remote,
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
with_deps=self.args.with_deps,
)
if st:
if self.args.quiet:
return 1
else:
self._show(st, indent)
else:
logger.info(self.UP_TO_DATE_MSG)
except Exception:
logger.error("failed to obtain data status")
return 1
return 0
def test_traceback(self):
stack_trace1 = 'stack_trace1\n'
stack_trace2 = 'stack_trace2\n'
try:
exc1 = Exception('exception1')
exc2 = DvcException('exception2', cause=exc1)
exc2.cause_tb = stack_trace1
exc3 = DvcException('exception3', cause=exc2)
exc3.cause_tb = stack_trace2
raise exc3
except Exception:
stack_trace3 = traceback.format_exc()
with logger.verbose():
logger.error('message')
output = ('Error: message - exception3: exception2: exception1\n'
'{line}\n'
'{stack_trace1}'
'\n'
'{stack_trace2}'
'\n'
'{stack_trace3}'
'{line}\n'
'\n'
'Having any troubles? Hit us up at https://dvc.org/support,'
' we are always happy to help!\n').format(
line='-' * 60,
stack_trace1=stack_trace1,
stack_trace2=stack_trace2,
stack_trace3=stack_trace3)
self.assertEqual(self.stderr, output)
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info["scheme"] != "local":
raise NotImplementedError
if from_info["scheme"] != "local":
raise NotImplementedError
logger.debug(
"Uploading '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
)
if not name:
name = os.path.basename(from_info["path"])
self._makedirs(to_info["path"])
try:
copyfile(from_info["path"], to_info["path"], name=name)
except Exception:
logger.error(
"failed to upload '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
)
def _show(self, section, opt):
try:
self.config.show(self.configobj, section, opt)
except DvcException:
logger.error("failed to show '{}'".format(self.args.name))
return 1
return 0
def run(self, unlock=False):
if not self.args.targets:
self.args.targets = self.default_targets
for target in self.args.targets:
try:
if self.args.ascii:
self._show_ascii(target, self.args.commands,
self.args.outs)
elif self.args.dot:
self.__write_dot(
target,
self.args.commands,
self.args.outs,
self.args.dot,
)
elif self.args.tree:
self._show_dependencies_tree(target, self.args.commands,
self.args.outs)
else:
self._show(target, self.args.commands, self.args.outs)
except DvcException:
msg = "failed to show pipeline for '{}'".format(target)
logger.error(msg)
return 1
return 0
def test_error(self, caplog):
with caplog.at_level(logging.INFO, logger="dvc"):
logger.error("message")
expected = "{red}ERROR{nc}: message\n".format(**colors)
assert expected == formatter.format(caplog.records[0])
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
gs = self.gs
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info["scheme"] != "gs":
raise NotImplementedError
if from_info["scheme"] != "local":
raise NotImplementedError
logger.debug("Uploading '{}' to '{}/{}'".format(
from_info["path"], to_info["bucket"], to_info["path"]))
if not name:
name = os.path.basename(from_info["path"])
progress.update_target(name, 0, None)
try:
bucket = gs.bucket(to_info["bucket"])
blob = bucket.blob(to_info["path"])
blob.upload_from_filename(from_info["path"])
except Exception:
msg = "failed to upload '{}' to '{}/{}'"
logger.error(
msg.format(from_info["path"], to_info["bucket"],
to_info["path"]))
continue
progress.finish_target(name)
def run(self):
overwrite = self.args.yes or self.args.overwrite_dvcfile
if not any([
self.args.deps, self.args.outs, self.args.outs_no_cache,
self.args.command
]): # pragma: no cover
logger.error("too few arguments. Specify at least one: '-d', "
"'-o', '-O', 'command'.")
return 1
try:
self.project.run(
cmd=self._parsed_cmd(),
outs=self.args.outs,
outs_no_cache=self.args.outs_no_cache,
metrics_no_cache=self.args.metrics_no_cache,
deps=self.args.deps,
fname=self.args.file,
cwd=self.args.cwd,
no_exec=self.args.no_exec,
overwrite=overwrite,
ignore_build_cache=self.args.ignore_build_cache,
remove_outs=self.args.remove_outs,
)
except DvcException:
logger.error("failed to run command")
return 1
return 0
def _spawn_posix(cmd):
# NOTE: using os._exit instead of sys.exit, because dvc built
# with PyInstaller has trouble with SystemExit exeption and throws
# errors such as "[26338] Failed to execute script __main__"
try:
pid = os.fork()
if pid > 0:
return
except OSError:
logger.error("failed at first fork")
os._exit(1) # pylint: disable=protected-access
os.setsid()
os.umask(0)
try:
pid = os.fork()
if pid > 0:
os._exit(0) # pylint: disable=protected-access
except OSError:
logger.error("failed at second fork")
os._exit(1) # pylint: disable=protected-access
sys.stdin.close()
sys.stdout.close()
sys.stderr.close()
Popen(cmd, env=fix_env(), close_fds=True, shell=False).communicate()
os._exit(0) # pylint: disable=protected-access
def run(self):
try:
# backward compatibility
if self.args.json_path:
typ = 'json'
xpath = self.args.json_path
elif self.args.tsv_path:
typ = 'tsv'
xpath = self.args.tsv_path
elif self.args.htsv_path:
typ = 'htsv'
xpath = self.args.htsv_path
elif self.args.csv_path:
typ = 'csv'
xpath = self.args.csv_path
elif self.args.hcsv_path:
typ = 'hcsv'
xpath = self.args.hcsv_path
else:
typ = self.args.type
xpath = self.args.xpath
self.project.metrics_show(self.args.path,
typ=typ,
xpath=xpath,
all_branches=self.args.all_branches,
all_tags=self.args.all_tags)
except DvcException:
logger.error('failed to show metrics')
return 1
return 0
def run_cmd(self):
try:
self.project.install()
except Exception:
logger.error('failed to install dvc hooks')
return 1
return 0
def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] != 'local':
raise NotImplementedError
if to_info['scheme'] != 'local':
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(
from_info['path'], to_info['path']))
if not name:
name = os.path.basename(to_info['path'])
self._makedirs(to_info['path'])
tmp_file = self.tmp_file(to_info['path'])
try:
copyfile(from_info['path'],
tmp_file,
no_progress_bar=no_progress_bar,
name=name)
except Exception:
logger.error("failed to download '{}' to '{}'".format(
from_info['path'], to_info['path']))
continue
os.rename(tmp_file, to_info['path'])
def run(self):
try:
# backward compatibility
if self.args.json_path:
typ = "json"
xpath = self.args.json_path
elif self.args.tsv_path:
typ = "tsv"
xpath = self.args.tsv_path
elif self.args.htsv_path:
typ = "htsv"
xpath = self.args.htsv_path
elif self.args.csv_path:
typ = "csv"
xpath = self.args.csv_path
elif self.args.hcsv_path:
typ = "hcsv"
xpath = self.args.hcsv_path
else:
typ = self.args.type
xpath = self.args.xpath
self.project.metrics_show(
self.args.path,
typ=typ,
xpath=xpath,
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
recursive=self.args.recursive,
)
except DvcException:
logger.error("failed to show metrics")
return 1
return 0
def run_cmd(self):
try:
self.repo.install()
except Exception:
logger.error("failed to install dvc hooks")
return 1
return 0
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
s3 = self.s3
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "s3":
raise NotImplementedError
if to_info["scheme"] == "s3":
self.copy(from_info, to_info, s3=s3)
continue
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(
from_info["bucket"], from_info["path"], to_info["path"]
)
logger.debug(msg)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
if no_progress_bar:
cb = None
else:
total = s3.head_object(
Bucket=from_info["bucket"], Key=from_info["path"]
)["ContentLength"]
cb = Callback(name, total)
s3.download_file(
from_info["bucket"],
from_info["path"],
tmp_file,
Callback=cb,
)
except Exception:
msg = "failed to download '{}/{}'".format(
from_info["bucket"], from_info["path"]
)
logger.error(msg)
continue
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def run(self):
for target in self.args.targets:
try:
self.project.add(target, recursive=self.args.recursive)
except DvcException:
logger.error('failed to add file')
return 1
return 0
def run(self):
try:
self.project.metrics_remove(self.args.path)
except DvcException:
msg = "failed to remove metric file '{}'".format(self.args.path)
logger.error(msg)
return 1
return 0
def run_cmd(self):
from dvc.lock import LockError
try:
with self.repo.lock:
return self.run()
except LockError:
logger.error("failed to lock before running a command")
return 1
def run(self):
try:
self.project.move(self.args.src, self.args.dst)
except DvcException:
msg = "failed to move '{}' -> '{}'".format(self.args.src,
self.args.dst)
logger.error(msg)
return 1
return 0
def test_logging_debug_with_datetime(caplog, dt):
with caplog.at_level(logging.DEBUG, logger="dvc"):
logger.warning("WARNING")
logger.debug("DEBUG")
logger.error("ERROR")
for record in caplog.records:
assert dt in formatter.format(record)
assert record.levelname == record.message
def run(self):
for target in self.args.targets:
try:
self.project.unprotect(target)
except DvcException:
msg = "failed to unprotect '{}'".format(target)
logger.error(msg)
return 1
return 0
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
ssh = self.ssh(
host=from_infos[0]["host"],
user=from_infos[0]["user"],
port=from_infos[0]["port"],
)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "ssh":
raise NotImplementedError
if to_info["scheme"] == "ssh":
assert from_info["host"] == to_info["host"]
assert from_info["port"] == to_info["port"]
assert from_info["user"] == to_info["user"]
self.copy(from_info, to_info, ssh=ssh)
continue
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(
from_info["host"], from_info["path"], to_info["path"]
)
logger.debug(msg)
if not name:
name = os.path.basename(to_info["path"])
self._makedirs(to_info["path"])
tmp_file = self.tmp_file(to_info["path"])
try:
ssh.open_sftp().get(
from_info["path"], tmp_file, callback=create_cb(name)
)
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.error(
msg.format(
from_info["host"], from_info["path"], to_info["path"]
)
)
continue
os.rename(tmp_file, to_info["path"])
progress.finish_target(name)
ssh.close()
def _set(self, section, opt, value):
try:
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException:
logger.error("failed to set '{}.{}' to '{}'".format(
section, opt, value))
return 1
return 0
def run(self):
for target in self.args.targets:
try:
outs_only = self._is_outs_only(target)
self.repo.remove(target, outs_only=outs_only)
except DvcException:
logger.error("failed to remove {}".format(target))
return 1
return 0
def test_error(self):
logger.error("message")
output = ("Error: message\n"
"\n"
"Having any troubles? Hit us up at https://dvc.org/support,"
" we are always happy to help!\n")
self.assertEqual(self.stderr, output)
