def get_dvc_info(): info = [ f"DVC version: {__version__} {package}", "---------------------------------", f"Platform: Python {platform.python_version()} on " f"{platform.platform()}", f"Supports:{_get_supported_remotes()}", ] try: with Repo() as repo: # cache_dir might not exist yet (e.g. after `dvc init`), and we # can't auto-create it, as it might cause issues if the user # later decides to enable shared cache mode with # `dvc config cache.shared group`. if os.path.exists(repo.odb.local.cache_dir): info.append(f"Cache types: {_get_linktype_support_info(repo)}") fs_type = get_fs_type(repo.odb.local.cache_dir) info.append(f"Cache directory: {fs_type}") else: info.append("Cache types: " + error_link("no-dvc-cache")) info.append(f"Caches: {_get_caches(repo.odb)}") info.append(f"Remotes: {_get_remotes(repo.config)}") root_directory = repo.root_dir fs_root = get_fs_type(os.path.abspath(root_directory)) info.append(f"Workspace directory: {fs_root}") info.append(f"Repo: {_get_dvc_repo_info(repo)}") except NotDvcRepoError: pass except SCMError: info.append("Repo: dvc, git (broken)") return "\n".join(info)
def _get_bucket(self, bucket): with self._get_s3() as s3: try: yield s3.Bucket(bucket) except s3.meta.client.exceptions.NoSuchBucket as exc: link = error_link("no-bucket") raise DvcException( f"Bucket '{bucket}' does not exist. {link}") from exc
def _get_s3(self): from botocore.exceptions import ( EndpointConnectionError, NoCredentialsError, ) try: yield self.s3 except NoCredentialsError as exc: link = error_link("no-credentials") raise DvcException( f"Unable to find AWS credentials. {link}") from exc except EndpointConnectionError as exc: link = error_link("connection-error") name = self.endpoint_url or "AWS S3" raise DvcException( f"Unable to connect to '{name}'. {link}") from exc
def __init__(self, target_infos, stats=None): from dvc.utils import error_link self.target_infos = target_infos self.stats = stats targets = [str(t) for t in target_infos] m = ("Checkout failed for following targets:\n{}\nIs your " "cache up to date?\n{}".format("\n".join(targets), error_link("missing-files"))) super().__init__(m)
def run(self): from dvc.repo import Repo package = PKG if PKG is None: package = "" else: package = f"({PKG})" info = [ f"DVC version: {__version__} {package}", "---------------------------------", f"Platform: Python {platform.python_version()} on " f"{platform.platform()}", f"Supports: {self.get_supported_remotes()}", ] try: repo = Repo() root_directory = repo.root_dir # cache_dir might not exist yet (e.g. after `dvc init`), and we # can't auto-create it, as it might cause issues if the user # later decides to enable shared cache mode with # `dvc config cache.shared group`. if os.path.exists(repo.cache.local.cache_dir): info.append("Cache types: {}".format( self.get_linktype_support_info(repo))) if psutil: fs_type = self.get_fs_type(repo.cache.local.cache_dir) info.append(f"Cache directory: {fs_type}") else: info.append("Cache types: " + error_link("no-dvc-cache")) except NotDvcRepoError: root_directory = os.getcwd() except SCMError: root_directory = os.getcwd() info.append("Repo: dvc, git (broken)") else: if psutil: fs_root = self.get_fs_type(os.path.abspath(root_directory)) info.append(f"Workspace directory: {fs_root}") info.append("Repo: {}".format(_get_dvc_repo_info(repo))) logger.info("\n".join(info)) return 0
def main(argv=None): # noqa: C901 """Run dvc CLI command. Args: argv: optional list of arguments to parse. sys.argv is used by default. Returns: int: command's return code. """ args = None disable_other_loggers() outerLogLevel = logger.level try: args = parse_args(argv) level = None if args.quiet: level = logging.CRITICAL elif args.verbose == 1: level = logging.DEBUG elif args.verbose > 1: level = logging.TRACE if level is not None: logger.setLevel(level) logger.trace(args) with profile(enable=args.cprofile, dump=args.cprofile_dump): cmd = args.func(args) ret = cmd.run() except ConfigError: logger.exception("configuration error") ret = 251 except KeyboardInterrupt: logger.exception("interrupted by the user") ret = 252 except NotDvcRepoError: logger.exception("") ret = 253 except DvcParserError: ret = 254 except DvcException: ret = 255 logger.exception("") except Exception as exc: # noqa, pylint: disable=broad-except # pylint: disable=no-member if isinstance(exc, OSError) and exc.errno == errno.EMFILE: logger.exception( "too many open files, please visit " "{} to see how to handle this " "problem".format(error_link("many-files")), extra={"tb_only": True}, ) else: logger.exception("unexpected error") logger.info(FOOTER) ret = 255 try: if analytics.is_enabled(): analytics.collect_and_send_report(args, ret) return ret finally: logger.setLevel(outerLogLevel) # Closing pools by-hand to prevent weird messages when closing SSH # connections. See https://github.com/iterative/dvc/issues/3248 for # more info. close_pools() # Remove cached repos in the end of the call, these are anonymous # so won't be reused by any other subsequent run anyway. clean_repos()
def main(argv=None): # noqa: C901 """Main entry point for dvc CLI. Args: argv: optional list of arguments to parse. sys.argv is used by default. Returns: int: command's return code. """ from dvc._debug import debugtools from dvc.config import ConfigError from dvc.exceptions import DvcException, NotDvcRepoError from dvc.logger import FOOTER, disable_other_loggers # NOTE: stderr/stdout may be closed if we are running from dvc.daemon. # On Linux we directly call cli.main after double forking and closing # the copied parent's standard file descriptors. If we make any logging # calls in this state it will cause an exception due to writing to a closed # file descriptor. if sys.stderr.closed: # pylint: disable=using-constant-test logging.disable() elif sys.stdout.closed: # pylint: disable=using-constant-test logging.disable(logging.INFO) args = None disable_other_loggers() outerLogLevel = logger.level try: args = parse_args(argv) level = None if args.quiet: level = logging.CRITICAL elif args.verbose == 1: level = logging.DEBUG elif args.verbose > 1: level = logging.TRACE if level is not None: logger.setLevel(level) logger.trace(args) if not sys.stdout.closed and not args.quiet: from dvc.ui import ui ui.enable() with debugtools(args): cmd = args.func(args) ret = cmd.do_run() except ConfigError: logger.exception("configuration error") ret = 251 except KeyboardInterrupt: logger.exception("interrupted by the user") ret = 252 except NotDvcRepoError: logger.exception("") ret = 253 except DvcException: ret = 255 logger.exception("") except DvcParserError: ret = 254 except Exception as exc: # noqa, pylint: disable=broad-except # pylint: disable=no-member import errno if isinstance(exc, OSError) and exc.errno == errno.EMFILE: from dvc.utils import error_link logger.exception( "too many open files, please visit " "{} to see how to handle this " "problem".format(error_link("many-files")), extra={"tb_only": True}, ) else: from dvc.info import get_dvc_info logger.exception("unexpected error") dvc_info = get_dvc_info() logger.debug("Version info for developers:\n%s", dvc_info) logger.info(FOOTER) ret = 255 try: from dvc import analytics if analytics.is_enabled(): analytics.collect_and_send_report(args, ret) return ret finally: logger.setLevel(outerLogLevel) from dvc.external_repo import clean_repos # Remove cached repos in the end of the call, these are anonymous # so won't be reused by any other subsequent run anyway. clean_repos()
def _log_exceptions(exc: Exception) -> Optional[int]: """Try to log some known exceptions, that are not DVCExceptions.""" from dvc.utils import error_link, format_link if isinstance(exc, OSError): import errno if exc.errno == errno.EMFILE: logger.exception( "too many open files, please visit " "{} to see how to handle this " "problem".format(error_link("many-files")), extra={"tb_only": True}, ) else: _log_unknown_exceptions() return None from dvc.fs import AuthError, ConfigError, RemoteMissingDepsError if isinstance(exc, RemoteMissingDepsError): from dvc.utils.pkg import PKG proto = exc.protocol by_pkg = { "pip": f"pip install 'dvc[{proto}]'", "conda": f"conda install -c conda-forge dvc-{proto}", } cmd = by_pkg.get(PKG) if cmd: link = format_link("https://dvc.org/doc/install") hint = (f"To install dvc with those dependencies, run:\n" "\n" f"\t{cmd}\n" "\n" f"See {link} for more info.") else: link = format_link("https://github.com/iterative/dvc/issues") hint = f"\nPlease report this bug to {link}. Thank you!" logger.exception( f"URL '{exc.url}' is supported but requires these missing " f"dependencies: {exc.missing_deps}. {hint}", extra={"tb_only": True}, ) return None if isinstance(exc, (AuthError, ConfigError)): link = format_link("https://man.dvc.org/remote/modify") logger.exception("configuration error") logger.exception( f"{exc!s}\nLearn more about configuration settings at {link}.", extra={"tb_only": True}, ) return 251 from dvc_data.hashfile.cache import DiskError if isinstance(exc, DiskError): from dvc.utils import relpath directory = relpath(exc.directory) logger.exception( f"Could not open pickled '{exc.type}' cache.\n" f"Remove the '{directory}' directory and then retry this command." f"\nSee {error_link('pickle')} for more information.", extra={"tb_only": True}, ) return None from dvc_data.stage import IgnoreInCollectedDirError if isinstance(exc, IgnoreInCollectedDirError): logger.exception("") return None _log_unknown_exceptions() return None
def main(argv=None): # noqa: C901 """Main entry point for dvc CLI. Args: argv: optional list of arguments to parse. sys.argv is used by default. Returns: int: command's return code. """ from dvc._debug import debugtools from dvc.config import ConfigError from dvc.exceptions import DvcException, NotDvcRepoError from dvc.logger import FOOTER, disable_other_loggers args = None disable_other_loggers() outerLogLevel = logger.level try: args = parse_args(argv) level = None if args.quiet: level = logging.CRITICAL elif args.verbose == 1: level = logging.DEBUG elif args.verbose > 1: level = logging.TRACE if level is not None: logger.setLevel(level) logger.trace(args) if not args.quiet: from dvc.ui import ui ui.enable() with debugtools(args): cmd = args.func(args) ret = cmd.do_run() except ConfigError: logger.exception("configuration error") ret = 251 except KeyboardInterrupt: logger.exception("interrupted by the user") ret = 252 except NotDvcRepoError: logger.exception("") ret = 253 except DvcException: ret = 255 logger.exception("") except DvcParserError: ret = 254 except Exception as exc: # noqa, pylint: disable=broad-except # pylint: disable=no-member import errno if isinstance(exc, OSError) and exc.errno == errno.EMFILE: from dvc.utils import error_link logger.exception( "too many open files, please visit " "{} to see how to handle this " "problem".format(error_link("many-files")), extra={"tb_only": True}, ) else: from dvc.info import get_dvc_info logger.exception("unexpected error") dvc_info = get_dvc_info() logger.debug("Version info for developers:\n%s", dvc_info) logger.info(FOOTER) ret = 255 try: from dvc import analytics if analytics.is_enabled(): analytics.collect_and_send_report(args, ret) return ret finally: logger.setLevel(outerLogLevel) from dvc.fs.pool import close_pools # Closing pools by-hand to prevent weird messages when closing SSH # connections. See https://github.com/iterative/dvc/issues/3248 for # more info. close_pools() from dvc.external_repo import clean_repos # Remove cached repos in the end of the call, these are anonymous # so won't be reused by any other subsequent run anyway. clean_repos()