def get_dvc_info():
info = [
f"DVC version: {__version__} {package}",
"---------------------------------",
f"Platform: Python {platform.python_version()} on "
f"{platform.platform()}",
f"Supports:{_get_supported_remotes()}",
]
try:
with Repo() as repo:
# cache_dir might not exist yet (e.g. after `dvc init`), and we
# can't auto-create it, as it might cause issues if the user
# later decides to enable shared cache mode with
# `dvc config cache.shared group`.
if os.path.exists(repo.odb.local.cache_dir):
info.append(f"Cache types: {_get_linktype_support_info(repo)}")
fs_type = get_fs_type(repo.odb.local.cache_dir)
info.append(f"Cache directory: {fs_type}")
else:
info.append("Cache types: " + error_link("no-dvc-cache"))
info.append(f"Caches: {_get_caches(repo.odb)}")
info.append(f"Remotes: {_get_remotes(repo.config)}")
root_directory = repo.root_dir
fs_root = get_fs_type(os.path.abspath(root_directory))
info.append(f"Workspace directory: {fs_root}")
info.append(f"Repo: {_get_dvc_repo_info(repo)}")
except NotDvcRepoError:
pass
except SCMError:
info.append("Repo: dvc, git (broken)")
return "\n".join(info)
def _get_bucket(self, bucket):
with self._get_s3() as s3:
try:
yield s3.Bucket(bucket)
except s3.meta.client.exceptions.NoSuchBucket as exc:
link = error_link("no-bucket")
raise DvcException(
f"Bucket '{bucket}' does not exist. {link}") from exc
def _get_s3(self):
from botocore.exceptions import (
EndpointConnectionError,
NoCredentialsError,
)
try:
yield self.s3
except NoCredentialsError as exc:
link = error_link("no-credentials")
raise DvcException(
f"Unable to find AWS credentials. {link}") from exc
except EndpointConnectionError as exc:
link = error_link("connection-error")
name = self.endpoint_url or "AWS S3"
raise DvcException(
f"Unable to connect to '{name}'. {link}") from exc
def __init__(self, target_infos, stats=None):
from dvc.utils import error_link
self.target_infos = target_infos
self.stats = stats
targets = [str(t) for t in target_infos]
m = ("Checkout failed for following targets:\n{}\nIs your "
"cache up to date?\n{}".format("\n".join(targets),
error_link("missing-files")))
super().__init__(m)
def run(self):
from dvc.repo import Repo
package = PKG
if PKG is None:
package = ""
else:
package = f"({PKG})"
info = [
f"DVC version: {__version__} {package}",
"---------------------------------",
f"Platform: Python {platform.python_version()} on "
f"{platform.platform()}",
f"Supports: {self.get_supported_remotes()}",
]
try:
repo = Repo()
root_directory = repo.root_dir
# cache_dir might not exist yet (e.g. after `dvc init`), and we
# can't auto-create it, as it might cause issues if the user
# later decides to enable shared cache mode with
# `dvc config cache.shared group`.
if os.path.exists(repo.cache.local.cache_dir):
info.append("Cache types: {}".format(
self.get_linktype_support_info(repo)))
if psutil:
fs_type = self.get_fs_type(repo.cache.local.cache_dir)
info.append(f"Cache directory: {fs_type}")
else:
info.append("Cache types: " + error_link("no-dvc-cache"))
except NotDvcRepoError:
root_directory = os.getcwd()
except SCMError:
root_directory = os.getcwd()
info.append("Repo: dvc, git (broken)")
else:
if psutil:
fs_root = self.get_fs_type(os.path.abspath(root_directory))
info.append(f"Workspace directory: {fs_root}")
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
logger.info("\n".join(info))
return 0
def main(argv=None): # noqa: C901
"""Run dvc CLI command.
Args:
argv: optional list of arguments to parse. sys.argv is used by default.
Returns:
int: command's return code.
"""
args = None
disable_other_loggers()
outerLogLevel = logger.level
try:
args = parse_args(argv)
level = None
if args.quiet:
level = logging.CRITICAL
elif args.verbose == 1:
level = logging.DEBUG
elif args.verbose > 1:
level = logging.TRACE
if level is not None:
logger.setLevel(level)
logger.trace(args)
with profile(enable=args.cprofile, dump=args.cprofile_dump):
cmd = args.func(args)
ret = cmd.run()
except ConfigError:
logger.exception("configuration error")
ret = 251
except KeyboardInterrupt:
logger.exception("interrupted by the user")
ret = 252
except NotDvcRepoError:
logger.exception("")
ret = 253
except DvcParserError:
ret = 254
except DvcException:
ret = 255
logger.exception("")
except Exception as exc: # noqa, pylint: disable=broad-except
# pylint: disable=no-member
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
logger.exception(
"too many open files, please visit "
"{} to see how to handle this "
"problem".format(error_link("many-files")),
extra={"tb_only": True},
)
else:
logger.exception("unexpected error")
logger.info(FOOTER)
ret = 255
try:
if analytics.is_enabled():
analytics.collect_and_send_report(args, ret)
return ret
finally:
logger.setLevel(outerLogLevel)
# Closing pools by-hand to prevent weird messages when closing SSH
# connections. See https://github.com/iterative/dvc/issues/3248 for
# more info.
close_pools()
# Remove cached repos in the end of the call, these are anonymous
# so won't be reused by any other subsequent run anyway.
clean_repos()
def main(argv=None): # noqa: C901
"""Main entry point for dvc CLI.
Args:
argv: optional list of arguments to parse. sys.argv is used by default.
Returns:
int: command's return code.
"""
from dvc._debug import debugtools
from dvc.config import ConfigError
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.logger import FOOTER, disable_other_loggers
# NOTE: stderr/stdout may be closed if we are running from dvc.daemon.
# On Linux we directly call cli.main after double forking and closing
# the copied parent's standard file descriptors. If we make any logging
# calls in this state it will cause an exception due to writing to a closed
# file descriptor.
if sys.stderr.closed: # pylint: disable=using-constant-test
logging.disable()
elif sys.stdout.closed: # pylint: disable=using-constant-test
logging.disable(logging.INFO)
args = None
disable_other_loggers()
outerLogLevel = logger.level
try:
args = parse_args(argv)
level = None
if args.quiet:
level = logging.CRITICAL
elif args.verbose == 1:
level = logging.DEBUG
elif args.verbose > 1:
level = logging.TRACE
if level is not None:
logger.setLevel(level)
logger.trace(args)
if not sys.stdout.closed and not args.quiet:
from dvc.ui import ui
ui.enable()
with debugtools(args):
cmd = args.func(args)
ret = cmd.do_run()
except ConfigError:
logger.exception("configuration error")
ret = 251
except KeyboardInterrupt:
logger.exception("interrupted by the user")
ret = 252
except NotDvcRepoError:
logger.exception("")
ret = 253
except DvcException:
ret = 255
logger.exception("")
except DvcParserError:
ret = 254
except Exception as exc: # noqa, pylint: disable=broad-except
# pylint: disable=no-member
import errno
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
from dvc.utils import error_link
logger.exception(
"too many open files, please visit "
"{} to see how to handle this "
"problem".format(error_link("many-files")),
extra={"tb_only": True},
)
else:
from dvc.info import get_dvc_info
logger.exception("unexpected error")
dvc_info = get_dvc_info()
logger.debug("Version info for developers:\n%s", dvc_info)
logger.info(FOOTER)
ret = 255
try:
from dvc import analytics
if analytics.is_enabled():
analytics.collect_and_send_report(args, ret)
return ret
finally:
logger.setLevel(outerLogLevel)
from dvc.external_repo import clean_repos
# Remove cached repos in the end of the call, these are anonymous
# so won't be reused by any other subsequent run anyway.
clean_repos()
def _log_exceptions(exc: Exception) -> Optional[int]:
"""Try to log some known exceptions, that are not DVCExceptions."""
from dvc.utils import error_link, format_link
if isinstance(exc, OSError):
import errno
if exc.errno == errno.EMFILE:
logger.exception(
"too many open files, please visit "
"{} to see how to handle this "
"problem".format(error_link("many-files")),
extra={"tb_only": True},
)
else:
_log_unknown_exceptions()
return None
from dvc.fs import AuthError, ConfigError, RemoteMissingDepsError
if isinstance(exc, RemoteMissingDepsError):
from dvc.utils.pkg import PKG
proto = exc.protocol
by_pkg = {
"pip": f"pip install 'dvc[{proto}]'",
"conda": f"conda install -c conda-forge dvc-{proto}",
}
cmd = by_pkg.get(PKG)
if cmd:
link = format_link("https://dvc.org/doc/install")
hint = (f"To install dvc with those dependencies, run:\n"
"\n"
f"\t{cmd}\n"
"\n"
f"See {link} for more info.")
else:
link = format_link("https://github.com/iterative/dvc/issues")
hint = f"\nPlease report this bug to {link}. Thank you!"
logger.exception(
f"URL '{exc.url}' is supported but requires these missing "
f"dependencies: {exc.missing_deps}. {hint}",
extra={"tb_only": True},
)
return None
if isinstance(exc, (AuthError, ConfigError)):
link = format_link("https://man.dvc.org/remote/modify")
logger.exception("configuration error")
logger.exception(
f"{exc!s}\nLearn more about configuration settings at {link}.",
extra={"tb_only": True},
)
return 251
from dvc_data.hashfile.cache import DiskError
if isinstance(exc, DiskError):
from dvc.utils import relpath
directory = relpath(exc.directory)
logger.exception(
f"Could not open pickled '{exc.type}' cache.\n"
f"Remove the '{directory}' directory and then retry this command."
f"\nSee {error_link('pickle')} for more information.",
extra={"tb_only": True},
)
return None
from dvc_data.stage import IgnoreInCollectedDirError
if isinstance(exc, IgnoreInCollectedDirError):
logger.exception("")
return None
_log_unknown_exceptions()
return None
def main(argv=None): # noqa: C901
"""Main entry point for dvc CLI.
Args:
argv: optional list of arguments to parse. sys.argv is used by default.
Returns:
int: command's return code.
"""
from dvc._debug import debugtools
from dvc.config import ConfigError
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.logger import FOOTER, disable_other_loggers
args = None
disable_other_loggers()
outerLogLevel = logger.level
try:
args = parse_args(argv)
level = None
if args.quiet:
level = logging.CRITICAL
elif args.verbose == 1:
level = logging.DEBUG
elif args.verbose > 1:
level = logging.TRACE
if level is not None:
logger.setLevel(level)
logger.trace(args)
if not args.quiet:
from dvc.ui import ui
ui.enable()
with debugtools(args):
cmd = args.func(args)
ret = cmd.do_run()
except ConfigError:
logger.exception("configuration error")
ret = 251
except KeyboardInterrupt:
logger.exception("interrupted by the user")
ret = 252
except NotDvcRepoError:
logger.exception("")
ret = 253
except DvcException:
ret = 255
logger.exception("")
except DvcParserError:
ret = 254
except Exception as exc: # noqa, pylint: disable=broad-except
# pylint: disable=no-member
import errno
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
from dvc.utils import error_link
logger.exception(
"too many open files, please visit "
"{} to see how to handle this "
"problem".format(error_link("many-files")),
extra={"tb_only": True},
)
else:
from dvc.info import get_dvc_info
logger.exception("unexpected error")
dvc_info = get_dvc_info()
logger.debug("Version info for developers:\n%s", dvc_info)
logger.info(FOOTER)
ret = 255
try:
from dvc import analytics
if analytics.is_enabled():
analytics.collect_and_send_report(args, ret)
return ret
finally:
logger.setLevel(outerLogLevel)
from dvc.fs.pool import close_pools
# Closing pools by-hand to prevent weird messages when closing SSH
# connections. See https://github.com/iterative/dvc/issues/3248 for
# more info.
close_pools()
from dvc.external_repo import clean_repos
# Remove cached repos in the end of the call, these are anonymous
# so won't be reused by any other subsequent run anyway.
clean_repos()
