def run(self): recursive = not self.args.single_item saved_dir = os.path.realpath(os.curdir) if self.args.cwd: os.chdir(self.args.cwd) # Dirty hack so the for loop below can at least enter once if self.args.all_pipelines: self.args.targets = [None] ret = 0 for target in self.args.targets: try: stages = self.project.reproduce( target, recursive=recursive, force=self.args.force, dry=self.args.dry, interactive=self.args.interactive, pipeline=self.args.pipeline, all_pipelines=self.args.all_pipelines, ignore_build_cache=self.args.ignore_build_cache) if len(stages) == 0: logger.info(CmdDataStatus.UP_TO_DATE_MSG) if self.args.metrics: self.project.metrics_show() except DvcException: logger.error() ret = 1 break os.chdir(saved_dir) return ret
def test_traceback(self): stack_trace1 = "stack_trace1\n" stack_trace2 = "stack_trace2\n" try: exc1 = Exception("exception1") exc2 = DvcException("exception2", cause=exc1) exc2.cause_tb = stack_trace1 exc3 = DvcException("exception3", cause=exc2) exc3.cause_tb = stack_trace2 raise exc3 except Exception: stack_trace3 = traceback.format_exc() with logger.verbose(): logger.error("message") output = ("Error: message - exception3: exception2: exception1\n" "{line}\n" "{stack_trace1}" "\n" "{stack_trace2}" "\n" "{stack_trace3}" "{line}\n" "\n" "Having any troubles? Hit us up at https://dvc.org/support," " we are always happy to help!\n").format( line="-" * 60, stack_trace1=stack_trace1, stack_trace2=stack_trace2, stack_trace3=stack_trace3, ) self.assertEqual(self.stderr, output)
def run(self): from dvc.remote import _get, RemoteLOCAL remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) if remote == RemoteLOCAL: self.args.url = self.resolve_path( self.args.url, self.configobj.filename ) section = Config.SECTION_REMOTE_FMT.format(self.args.name) if (section in self.configobj.keys()) and not self.args.force: logger.error( "Remote with name {} already exists. " "Use -f (--force) to overwrite remote " "with new value".format(self.args.name) ) return 1 ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) if ret != 0: return ret if self.args.default: msg = "Setting '{}' as a default remote.".format(self.args.name) logger.info(msg) ret = self._set( Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name ) return ret
def load_dir_cache(self, md5): path = self.get(md5) assert self.is_dir_cache(path) dir_info = self._dir_info.get(md5) if dir_info: return dir_info try: with open(path, "r") as fd: d = json.load(fd) except Exception: msg = "Failed to load dir cache '{}'" logger.error(msg.format(os.path.relpath(path))) return [] if not isinstance(d, list): msg = "dir cache file format error '{}' [skipping the file]" logger.error(msg.format(os.path.relpath(path))) return [] for info in d: info["relpath"] = self.to_ospath(info["relpath"]) self._dir_info[md5] = d return d
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) s3 = self.s3 for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info["scheme"] != "s3": raise NotImplementedError if from_info["scheme"] != "local": raise NotImplementedError logger.debug("Uploading '{}' to '{}/{}'".format( from_info["path"], to_info["bucket"], to_info["path"])) if not name: name = os.path.basename(from_info["path"]) total = os.path.getsize(from_info["path"]) cb = Callback(name, total) try: s3.upload_file(from_info["path"], to_info["bucket"], to_info["path"], Callback=cb) except Exception: msg = "failed to upload '{}'".format(from_info["path"]) logger.error(msg) continue progress.finish_target(name)
def do_run(self, target=None): indent = 1 if self.args.cloud else 0 try: st = self.project.status( target=target, jobs=self.args.jobs, cloud=self.args.cloud, show_checksums=self.args.show_checksums, remote=self.args.remote, all_branches=self.args.all_branches, all_tags=self.args.all_tags, with_deps=self.args.with_deps, ) if st: if self.args.quiet: return 1 else: self._show(st, indent) else: logger.info(self.UP_TO_DATE_MSG) except Exception: logger.error("failed to obtain data status") return 1 return 0
def test_traceback(self): stack_trace1 = 'stack_trace1\n' stack_trace2 = 'stack_trace2\n' try: exc1 = Exception('exception1') exc2 = DvcException('exception2', cause=exc1) exc2.cause_tb = stack_trace1 exc3 = DvcException('exception3', cause=exc2) exc3.cause_tb = stack_trace2 raise exc3 except Exception: stack_trace3 = traceback.format_exc() with logger.verbose(): logger.error('message') output = ('Error: message - exception3: exception2: exception1\n' '{line}\n' '{stack_trace1}' '\n' '{stack_trace2}' '\n' '{stack_trace3}' '{line}\n' '\n' 'Having any troubles? Hit us up at https://dvc.org/support,' ' we are always happy to help!\n').format( line='-' * 60, stack_trace1=stack_trace1, stack_trace2=stack_trace2, stack_trace3=stack_trace3) self.assertEqual(self.stderr, output)
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info["scheme"] != "local": raise NotImplementedError if from_info["scheme"] != "local": raise NotImplementedError logger.debug( "Uploading '{}' to '{}'".format( from_info["path"], to_info["path"] ) ) if not name: name = os.path.basename(from_info["path"]) self._makedirs(to_info["path"]) try: copyfile(from_info["path"], to_info["path"], name=name) except Exception: logger.error( "failed to upload '{}' to '{}'".format( from_info["path"], to_info["path"] ) )
def _show(self, section, opt): try: self.config.show(self.configobj, section, opt) except DvcException: logger.error("failed to show '{}'".format(self.args.name)) return 1 return 0
def run(self, unlock=False): if not self.args.targets: self.args.targets = self.default_targets for target in self.args.targets: try: if self.args.ascii: self._show_ascii(target, self.args.commands, self.args.outs) elif self.args.dot: self.__write_dot( target, self.args.commands, self.args.outs, self.args.dot, ) elif self.args.tree: self._show_dependencies_tree(target, self.args.commands, self.args.outs) else: self._show(target, self.args.commands, self.args.outs) except DvcException: msg = "failed to show pipeline for '{}'".format(target) logger.error(msg) return 1 return 0
def test_error(self, caplog): with caplog.at_level(logging.INFO, logger="dvc"): logger.error("message") expected = "{red}ERROR{nc}: message\n".format(**colors) assert expected == formatter.format(caplog.records[0])
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) gs = self.gs for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info["scheme"] != "gs": raise NotImplementedError if from_info["scheme"] != "local": raise NotImplementedError logger.debug("Uploading '{}' to '{}/{}'".format( from_info["path"], to_info["bucket"], to_info["path"])) if not name: name = os.path.basename(from_info["path"]) progress.update_target(name, 0, None) try: bucket = gs.bucket(to_info["bucket"]) blob = bucket.blob(to_info["path"]) blob.upload_from_filename(from_info["path"]) except Exception: msg = "failed to upload '{}' to '{}/{}'" logger.error( msg.format(from_info["path"], to_info["bucket"], to_info["path"])) continue progress.finish_target(name)
def run(self): overwrite = self.args.yes or self.args.overwrite_dvcfile if not any([ self.args.deps, self.args.outs, self.args.outs_no_cache, self.args.command ]): # pragma: no cover logger.error("too few arguments. Specify at least one: '-d', " "'-o', '-O', 'command'.") return 1 try: self.project.run( cmd=self._parsed_cmd(), outs=self.args.outs, outs_no_cache=self.args.outs_no_cache, metrics_no_cache=self.args.metrics_no_cache, deps=self.args.deps, fname=self.args.file, cwd=self.args.cwd, no_exec=self.args.no_exec, overwrite=overwrite, ignore_build_cache=self.args.ignore_build_cache, remove_outs=self.args.remove_outs, ) except DvcException: logger.error("failed to run command") return 1 return 0
def _spawn_posix(cmd): # NOTE: using os._exit instead of sys.exit, because dvc built # with PyInstaller has trouble with SystemExit exeption and throws # errors such as "[26338] Failed to execute script __main__" try: pid = os.fork() if pid > 0: return except OSError: logger.error("failed at first fork") os._exit(1) # pylint: disable=protected-access os.setsid() os.umask(0) try: pid = os.fork() if pid > 0: os._exit(0) # pylint: disable=protected-access except OSError: logger.error("failed at second fork") os._exit(1) # pylint: disable=protected-access sys.stdin.close() sys.stdout.close() sys.stderr.close() Popen(cmd, env=fix_env(), close_fds=True, shell=False).communicate() os._exit(0) # pylint: disable=protected-access
def run(self): try: # backward compatibility if self.args.json_path: typ = 'json' xpath = self.args.json_path elif self.args.tsv_path: typ = 'tsv' xpath = self.args.tsv_path elif self.args.htsv_path: typ = 'htsv' xpath = self.args.htsv_path elif self.args.csv_path: typ = 'csv' xpath = self.args.csv_path elif self.args.hcsv_path: typ = 'hcsv' xpath = self.args.hcsv_path else: typ = self.args.type xpath = self.args.xpath self.project.metrics_show(self.args.path, typ=typ, xpath=xpath, all_branches=self.args.all_branches, all_tags=self.args.all_tags) except DvcException: logger.error('failed to show metrics') return 1 return 0
def run_cmd(self): try: self.project.install() except Exception: logger.error('failed to install dvc hooks') return 1 return 0
def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] != 'local': raise NotImplementedError if to_info['scheme'] != 'local': raise NotImplementedError logger.debug("Downloading '{}' to '{}'".format( from_info['path'], to_info['path'])) if not name: name = os.path.basename(to_info['path']) self._makedirs(to_info['path']) tmp_file = self.tmp_file(to_info['path']) try: copyfile(from_info['path'], tmp_file, no_progress_bar=no_progress_bar, name=name) except Exception: logger.error("failed to download '{}' to '{}'".format( from_info['path'], to_info['path'])) continue os.rename(tmp_file, to_info['path'])
def run(self): try: # backward compatibility if self.args.json_path: typ = "json" xpath = self.args.json_path elif self.args.tsv_path: typ = "tsv" xpath = self.args.tsv_path elif self.args.htsv_path: typ = "htsv" xpath = self.args.htsv_path elif self.args.csv_path: typ = "csv" xpath = self.args.csv_path elif self.args.hcsv_path: typ = "hcsv" xpath = self.args.hcsv_path else: typ = self.args.type xpath = self.args.xpath self.project.metrics_show( self.args.path, typ=typ, xpath=xpath, all_branches=self.args.all_branches, all_tags=self.args.all_tags, recursive=self.args.recursive, ) except DvcException: logger.error("failed to show metrics") return 1 return 0
def run_cmd(self): try: self.repo.install() except Exception: logger.error("failed to install dvc hooks") return 1 return 0
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) s3 = self.s3 for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info["scheme"] != "s3": raise NotImplementedError if to_info["scheme"] == "s3": self.copy(from_info, to_info, s3=s3) continue if to_info["scheme"] != "local": raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format( from_info["bucket"], from_info["path"], to_info["path"] ) logger.debug(msg) tmp_file = tmp_fname(to_info["path"]) if not name: name = os.path.basename(to_info["path"]) makedirs(os.path.dirname(to_info["path"]), exist_ok=True) try: if no_progress_bar: cb = None else: total = s3.head_object( Bucket=from_info["bucket"], Key=from_info["path"] )["ContentLength"] cb = Callback(name, total) s3.download_file( from_info["bucket"], from_info["path"], tmp_file, Callback=cb, ) except Exception: msg = "failed to download '{}/{}'".format( from_info["bucket"], from_info["path"] ) logger.error(msg) continue move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name)
def run(self): for target in self.args.targets: try: self.project.add(target, recursive=self.args.recursive) except DvcException: logger.error('failed to add file') return 1 return 0
def run(self): try: self.project.metrics_remove(self.args.path) except DvcException: msg = "failed to remove metric file '{}'".format(self.args.path) logger.error(msg) return 1 return 0
def run_cmd(self): from dvc.lock import LockError try: with self.repo.lock: return self.run() except LockError: logger.error("failed to lock before running a command") return 1
def run(self): try: self.project.move(self.args.src, self.args.dst) except DvcException: msg = "failed to move '{}' -> '{}'".format(self.args.src, self.args.dst) logger.error(msg) return 1 return 0
def test_logging_debug_with_datetime(caplog, dt): with caplog.at_level(logging.DEBUG, logger="dvc"): logger.warning("WARNING") logger.debug("DEBUG") logger.error("ERROR") for record in caplog.records: assert dt in formatter.format(record) assert record.levelname == record.message
def run(self): for target in self.args.targets: try: self.project.unprotect(target) except DvcException: msg = "failed to unprotect '{}'".format(target) logger.error(msg) return 1 return 0
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) ssh = self.ssh( host=from_infos[0]["host"], user=from_infos[0]["user"], port=from_infos[0]["port"], ) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info["scheme"] != "ssh": raise NotImplementedError if to_info["scheme"] == "ssh": assert from_info["host"] == to_info["host"] assert from_info["port"] == to_info["port"] assert from_info["user"] == to_info["user"] self.copy(from_info, to_info, ssh=ssh) continue if to_info["scheme"] != "local": raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format( from_info["host"], from_info["path"], to_info["path"] ) logger.debug(msg) if not name: name = os.path.basename(to_info["path"]) self._makedirs(to_info["path"]) tmp_file = self.tmp_file(to_info["path"]) try: ssh.open_sftp().get( from_info["path"], tmp_file, callback=create_cb(name) ) except Exception: msg = "failed to download '{}/{}' to '{}'" logger.error( msg.format( from_info["host"], from_info["path"], to_info["path"] ) ) continue os.rename(tmp_file, to_info["path"]) progress.finish_target(name) ssh.close()
def _set(self, section, opt, value): try: self.config.set(self.configobj, section, opt, value) self.config.save(self.configobj) except DvcException: logger.error("failed to set '{}.{}' to '{}'".format( section, opt, value)) return 1 return 0
def run(self): for target in self.args.targets: try: outs_only = self._is_outs_only(target) self.repo.remove(target, outs_only=outs_only) except DvcException: logger.error("failed to remove {}".format(target)) return 1 return 0
def test_error(self): logger.error("message") output = ("Error: message\n" "\n" "Having any troubles? Hit us up at https://dvc.org/support," " we are always happy to help!\n") self.assertEqual(self.stderr, output)