def test_wrong_type_add(self): with open("metric.unknown", "w+") as fd: fd.write("unknown") fd.flush() ret = main(["add", "metric.unknown"]) self.assertEqual(ret, 0) with MockLoggerHandlers(logger.logger): logger.logger.handlers[1].stream = StringIO() ret = main(["metrics", "add", "metric.unknown", "-t", "unknown"]) self.assertEqual(ret, 1) self.assertIn( "failed to add metric file 'metric.unknown' - metric type " "'unknown' is not supported, must be one of " "[raw, json, csv, tsv, hcsv, htsv]", logger.logger.handlers[1].stream.getvalue(), ) ret = main(["metrics", "add", "metric.unknown", "-t", "raw"]) self.assertEqual(ret, 0) logger.logger.handlers[0].stream = StringIO() ret = main(["metrics", "show", "metric.unknown"]) self.assertEqual(ret, 0) self.assertIn( "\tmetric.unknown: unknown", logger.logger.handlers[0].stream.getvalue(), )
def test_wrong_type_modify(self): with open("metric.unknown", "w+") as fd: fd.write("unknown") fd.flush() ret = main(["run", "-m", "metric.unknown"]) self.assertEqual(ret, 0) with MockLoggerHandlers(logger.logger): logger.logger.handlers[1].stream = StringIO() ret = main( ["metrics", "modify", "metric.unknown", "-t", "unknown"]) self.assertEqual(ret, 1) self.assertIn( "failed to modify metric file settings - metric type 'unknown'" " is not supported, must be one of [raw, json, csv, tsv, hcsv, htsv]", logger.logger.handlers[1].stream.getvalue(), ) ret = main(["metrics", "modify", "metric.unknown", "-t", "CSV"]) self.assertEqual(ret, 0) logger.logger.handlers[0].stream = StringIO() ret = main(["metrics", "show", "metric.unknown"]) self.assertEqual(ret, 0) self.assertIn( "\tmetric.unknown: unknown", logger.logger.handlers[0].stream.getvalue(), )
def setUp(self): logger.logger.handlers = [ logger.logging.StreamHandler(), logger.logging.StreamHandler(), ] logger.logger.handlers[0].stream = StringIO() logger.logger.handlers[1].stream = StringIO() logger.set_default_level() self.color_patch.start()
def setUp(self): logger.logger.handlers = [ logger.logging.StreamHandler(), logger.logging.StreamHandler(), ] logger.logger.handlers[0].stream = StringIO() logger.logger.handlers[1].stream = StringIO() logger.set_default_level() self.consoleColorRemover = ConsoleFontColorsRemover() self.consoleColorRemover.__enter__()
def __write_dot(self, target, commands, outs): from dvc.utils.compat import StringIO import networkx from networkx.drawing.nx_pydot import write_dot _, edges, _ = self.__build_graph(target, commands, outs) edges = [edge[::-1] for edge in edges] simple_g = networkx.DiGraph() simple_g.add_edges_from(edges) dot_file = StringIO() write_dot(simple_g, dot_file) logger.info(dot_file.getvalue())
def _test(self): url = get_local_url() self.main(["remote", "add", "-d", TEST_REMOTE, url]) self.dvc.add(self.FOO) stage = self.dvc.run(deps=["foo"], outs=["bar"], cmd="echo bar > bar") self.main(["push"]) stage_file_path = stage.relpath with open(stage_file_path, "r") as stage_file: content = yaml.safe_load(stage_file) del (content["outs"][0]["md5"]) with open(stage_file_path, "w") as stage_file: yaml.dump(content, stage_file) with MockLoggerHandlers(logger.logger): logger.logger.handlers[0].stream = StringIO() self.main(["status", "-c"]) self.assertIn( "Warning: Output 'bar'(Stage: 'bar.dvc') is " "missing version info. Cache for it will not be " "collected. Use dvc repro to get your pipeline up to " "date.", logger.logger.handlers[0].stream.getvalue(), )
def _format_csv(content, delimiter): """Format delimited text to have same column width. Args: content (str): The content of a metric. delimiter (str): Value separator Returns: str: Formatted content. Example: >>> content = ( "value_mse,deviation_mse,data_set\n" "0.421601,0.173461,train\n" "0.67528,0.289545,testing\n" "0.671502,0.297848,validation\n" ) >>> _format_csv(content, ",") "value_mse deviation_mse data_set\n" "0.421601 0.173461 train\n" "0.67528 0.289545 testing\n" "0.671502 0.297848 validation\n" """ reader = csv_reader(StringIO(content), delimiter=builtin_str(delimiter)) rows = [row for row in reader] max_widths = [max(map(len, column)) for column in zip(*rows)] lines = [ " ".join("{entry:{width}}".format(entry=entry, width=width + 2) for entry, width in zip(row, max_widths)) for row in rows ] return "\n".join(lines)
def test_wrong_type_show(self): with open("metric.unknown", "w+") as fd: fd.write("unknown") fd.flush() ret = main(["run", "-m", "metric.unknown"]) self.assertEqual(ret, 0) with MockLoggerHandlers(logger.logger): logger.logger.handlers[0].stream = StringIO() ret = main( [ "metrics", "show", "metric.unknown", "-t", "unknown", "-x", "0,0", ] ) self.assertEqual(ret, 0) self.assertIn( "\tmetric.unknown: unknown", logger.logger.handlers[0].stream.getvalue(), )
def _load_user_ssh_config(hostname): user_config_file = RemoteSSH.ssh_config_filename() user_ssh_config = dict() if hostname and os.path.exists(user_config_file): ssh_config = paramiko.SSHConfig() with open(user_config_file) as f: # For whatever reason parsing directly from f is unreliable f_copy = StringIO(f.read()) ssh_config.parse(f_copy) user_ssh_config = ssh_config.lookup(hostname) return user_ssh_config
def _walk_exc(self, exc_info): import traceback buffer = StringIO() traceback.print_exception(*exc_info, file=buffer) exc = exc_info[1] tb = buffer.getvalue() exc_list = [str(exc)] tb_list = [tb] # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info while hasattr(exc, "cause") and exc.cause: exc_list.append(str(exc.cause)) if hasattr(exc, "cause_tb") and exc.cause_tb: tb_list.insert(0, str(exc.cause_tb)) exc = exc.cause return exc_list, tb_list
def _test(self, imp_urlparse_patch): logger.logger.handlers[1].stream = StringIO() page_address = "http://somesite.com/file_name" def dvc_exception(*args, **kwargs): raise DvcException("message") imp_urlparse_patch.side_effect = dvc_exception main(["import", page_address]) self.assertIn( "Error: failed to import " "http://somesite.com/file_name. You could also try " "downloading it manually and adding it with `dvc add` " "command.", logger.logger.handlers[1].stream.getvalue(), )
def open(self, path): relpath = os.path.relpath(path, self.git.working_dir) obj = self.git_object_by_path(path) if obj is None: msg = "No such file in branch '{}'".format(self.rev) raise IOError(errno.ENOENT, msg, relpath) if obj.mode == GIT_MODE_DIR: raise IOError(errno.EISDIR, "Is a directory", relpath) # GitPython's obj.data_stream is a fragile thing, it is better to # read it immediately, also it needs to be to decoded if we follow # the `open()` behavior (since data_stream.read() returns bytes, # and `open` with default "r" mode returns str) return StringIO(obj.data_stream.read().decode("utf-8"))
def test(self): with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): logger.handlers[1].stream = StringIO() ret = main(["add", self.FOO]) self.assertEqual(0, ret) foo_stage = os.path.abspath(self.FOO + Stage.STAGE_FILE_SUFFIX) # corrupt stage file with open(foo_stage, "a+") as file: file.write("this will break yaml file structure") ret = main(["add", self.BAR]) self.assertEqual(1, ret) self.assertIn( "unable to read stage file: {} " "YAML file structure is corrupted".format(foo_stage), logger.handlers[1].stream.getvalue(), )
def to_yaml_string(data): stream = StringIO() yaml = YAML() yaml.default_flow_style = False yaml.dump(data, stream) return stream.getvalue()
def from_yaml_string(s): return YAML().load(StringIO(s))
def reset_logger_standard_output(): from dvc.logger import logger logger.handlers[0].stream = StringIO()
def reset_logger_error_output(): from dvc.logger import logger logger.handlers[1].stream = StringIO()